2024-11-10 20:25:38,195 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-10 20:25:38,212 main DEBUG Took 0.014117 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 20:25:38,212 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 20:25:38,213 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 20:25:38,214 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 20:25:38,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,226 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 20:25:38,243 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,246 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,247 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,248 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,249 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,249 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,250 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,250 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,251 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,252 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,253 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,253 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,254 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,254 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,255 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,255 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,256 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,257 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,257 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,258 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 20:25:38,259 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,259 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 20:25:38,261 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 20:25:38,263 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 20:25:38,264 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 20:25:38,265 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 20:25:38,266 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 20:25:38,266 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 20:25:38,275 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 20:25:38,277 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 20:25:38,279 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 20:25:38,279 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 20:25:38,280 main DEBUG createAppenders(={Console}) 2024-11-10 20:25:38,281 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-11-10 20:25:38,281 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-10 20:25:38,281 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-11-10 20:25:38,282 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 20:25:38,282 main DEBUG OutputStream closed 2024-11-10 20:25:38,282 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 20:25:38,282 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 20:25:38,283 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-11-10 20:25:38,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 20:25:38,356 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 20:25:38,357 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 20:25:38,358 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 20:25:38,359 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 20:25:38,359 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 20:25:38,359 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 20:25:38,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 20:25:38,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 20:25:38,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 20:25:38,360 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 20:25:38,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 20:25:38,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 20:25:38,361 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 20:25:38,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 20:25:38,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 20:25:38,362 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 20:25:38,363 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 20:25:38,365 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 20:25:38,366 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-11-10 20:25:38,366 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 20:25:38,367 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-11-10T20:25:38,602 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0 2024-11-10 20:25:38,605 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 20:25:38,606 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T20:25:38,615 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-11-10T20:25:38,651 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=0, ProcessCount=11, AvailableMemoryMB=8432 2024-11-10T20:25:38,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T20:25:38,673 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08, deleteOnExit=true 2024-11-10T20:25:38,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T20:25:38,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/test.cache.data in system properties and HBase conf 2024-11-10T20:25:38,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T20:25:38,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.log.dir in system properties and HBase conf 2024-11-10T20:25:38,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T20:25:38,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T20:25:38,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T20:25:38,759 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T20:25:38,852 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T20:25:38,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:25:38,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:25:38,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T20:25:38,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:25:38,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T20:25:38,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T20:25:38,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:25:38,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:25:38,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T20:25:38,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/nfs.dump.dir in system properties and HBase conf 2024-11-10T20:25:38,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/java.io.tmpdir in system properties and HBase conf 2024-11-10T20:25:38,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:25:38,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T20:25:38,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T20:25:39,356 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:25:39,710 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T20:25:39,795 INFO [Time-limited test {}] log.Log(170): Logging initialized @2268ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T20:25:39,875 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:25:39,940 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:25:39,961 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:25:39,961 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:25:39,963 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:25:39,976 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:25:39,979 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:25:39,980 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:25:40,171 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/java.io.tmpdir/jetty-localhost-36455-hadoop-hdfs-3_4_1-tests_jar-_-any-14022981083850754242/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:25:40,178 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:36455} 2024-11-10T20:25:40,179 INFO [Time-limited test {}] server.Server(415): Started @2653ms 2024-11-10T20:25:40,204 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:25:40,585 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:25:40,591 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:25:40,592 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:25:40,593 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:25:40,593 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:25:40,597 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:25:40,597 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:25:40,718 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/java.io.tmpdir/jetty-localhost-46345-hadoop-hdfs-3_4_1-tests_jar-_-any-14517288742919512276/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:25:40,719 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:46345} 2024-11-10T20:25:40,719 INFO [Time-limited test {}] server.Server(415): Started @3193ms 2024-11-10T20:25:40,775 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:25:40,908 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:25:40,914 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:25:40,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:25:40,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:25:40,917 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:25:40,918 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:25:40,919 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:25:41,042 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/java.io.tmpdir/jetty-localhost-33947-hadoop-hdfs-3_4_1-tests_jar-_-any-16573195014149620971/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:25:41,043 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:33947} 2024-11-10T20:25:41,044 INFO [Time-limited test {}] server.Server(415): Started @3518ms 2024-11-10T20:25:41,046 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:25:41,221 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/data/data1/current/BP-1055372608-172.17.0.2-1731270339452/current, will proceed with Du for space computation calculation, 2024-11-10T20:25:41,221 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/data/data2/current/BP-1055372608-172.17.0.2-1731270339452/current, will proceed with Du for space computation calculation, 2024-11-10T20:25:41,221 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/data/data4/current/BP-1055372608-172.17.0.2-1731270339452/current, will proceed with Du for space computation calculation, 2024-11-10T20:25:41,221 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/data/data3/current/BP-1055372608-172.17.0.2-1731270339452/current, will proceed with Du for space computation calculation, 2024-11-10T20:25:41,278 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:25:41,279 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:25:41,350 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed5daa1ed2c94152 with lease ID 0x6e1f0719109d9657: Processing first storage report for DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4 from datanode DatanodeRegistration(127.0.0.1:33157, datanodeUuid=88bd4023-f81d-41ab-b25e-705b0871e99f, infoPort=45365, infoSecurePort=0, ipcPort=35877, storageInfo=lv=-57;cid=testClusterID;nsid=464730559;c=1731270339452) 2024-11-10T20:25:41,352 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed5daa1ed2c94152 with lease ID 0x6e1f0719109d9657: from storage DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4 node DatanodeRegistration(127.0.0.1:33157, datanodeUuid=88bd4023-f81d-41ab-b25e-705b0871e99f, infoPort=45365, infoSecurePort=0, ipcPort=35877, storageInfo=lv=-57;cid=testClusterID;nsid=464730559;c=1731270339452), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-10T20:25:41,353 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec875ae10ad576f2 with lease ID 0x6e1f0719109d9658: Processing first storage report for DS-222a65c5-269a-469b-90f5-9489587483a7 from datanode DatanodeRegistration(127.0.0.1:41181, datanodeUuid=dbd83278-d45c-4301-91ee-84deff001f20, infoPort=40219, infoSecurePort=0, ipcPort=33125, storageInfo=lv=-57;cid=testClusterID;nsid=464730559;c=1731270339452) 2024-11-10T20:25:41,353 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec875ae10ad576f2 with lease ID 0x6e1f0719109d9658: from storage DS-222a65c5-269a-469b-90f5-9489587483a7 node DatanodeRegistration(127.0.0.1:41181, datanodeUuid=dbd83278-d45c-4301-91ee-84deff001f20, infoPort=40219, infoSecurePort=0, ipcPort=33125, storageInfo=lv=-57;cid=testClusterID;nsid=464730559;c=1731270339452), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T20:25:41,354 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed5daa1ed2c94152 with lease ID 0x6e1f0719109d9657: Processing first storage report for DS-6c555d1c-5019-401a-baa2-5f17973d5115 from datanode DatanodeRegistration(127.0.0.1:33157, datanodeUuid=88bd4023-f81d-41ab-b25e-705b0871e99f, infoPort=45365, infoSecurePort=0, ipcPort=35877, storageInfo=lv=-57;cid=testClusterID;nsid=464730559;c=1731270339452) 2024-11-10T20:25:41,354 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed5daa1ed2c94152 with lease ID 0x6e1f0719109d9657: from storage DS-6c555d1c-5019-401a-baa2-5f17973d5115 node DatanodeRegistration(127.0.0.1:33157, datanodeUuid=88bd4023-f81d-41ab-b25e-705b0871e99f, infoPort=45365, infoSecurePort=0, ipcPort=35877, storageInfo=lv=-57;cid=testClusterID;nsid=464730559;c=1731270339452), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:25:41,354 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec875ae10ad576f2 with lease ID 0x6e1f0719109d9658: Processing first storage report for DS-d7a20635-848d-46dc-bff3-f7e3a357b15b from datanode DatanodeRegistration(127.0.0.1:41181, datanodeUuid=dbd83278-d45c-4301-91ee-84deff001f20, infoPort=40219, infoSecurePort=0, ipcPort=33125, storageInfo=lv=-57;cid=testClusterID;nsid=464730559;c=1731270339452) 2024-11-10T20:25:41,354 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec875ae10ad576f2 with lease ID 0x6e1f0719109d9658: from storage DS-d7a20635-848d-46dc-bff3-f7e3a357b15b node DatanodeRegistration(127.0.0.1:41181, datanodeUuid=dbd83278-d45c-4301-91ee-84deff001f20, infoPort=40219, infoSecurePort=0, ipcPort=33125, storageInfo=lv=-57;cid=testClusterID;nsid=464730559;c=1731270339452), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:25:41,468 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0 2024-11-10T20:25:41,543 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/zookeeper_0, clientPort=49357, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T20:25:41,553 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49357 2024-11-10T20:25:41,567 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:25:41,570 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:25:41,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:25:41,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:25:42,221 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b with version=8 2024-11-10T20:25:42,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/hbase-staging 2024-11-10T20:25:42,326 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T20:25:42,588 INFO [Time-limited test {}] client.ConnectionUtils(128): master/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:25:42,599 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:25:42,600 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:25:42,605 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:25:42,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:25:42,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:25:42,746 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T20:25:42,807 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T20:25:42,816 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T20:25:42,820 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:25:42,847 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3938 (auto-detected) 2024-11-10T20:25:42,849 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T20:25:42,868 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45501 2024-11-10T20:25:42,893 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45501 connecting to ZooKeeper ensemble=127.0.0.1:49357 2024-11-10T20:25:42,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:455010x0, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:25:42,928 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45501-0x10193dc68fd0000 connected 2024-11-10T20:25:42,954 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:25:42,956 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:25:42,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:25:42,971 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b, hbase.cluster.distributed=false 2024-11-10T20:25:42,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:25:43,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45501 2024-11-10T20:25:43,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45501 2024-11-10T20:25:43,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45501 2024-11-10T20:25:43,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45501 2024-11-10T20:25:43,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45501 2024-11-10T20:25:43,122 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:25:43,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:25:43,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:25:43,124 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:25:43,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:25:43,125 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:25:43,128 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T20:25:43,130 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:25:43,131 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40573 2024-11-10T20:25:43,133 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40573 connecting to ZooKeeper ensemble=127.0.0.1:49357 2024-11-10T20:25:43,134 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:25:43,138 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:25:43,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405730x0, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:25:43,146 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40573-0x10193dc68fd0001 connected 2024-11-10T20:25:43,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:25:43,150 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T20:25:43,158 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T20:25:43,160 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T20:25:43,165 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:25:43,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40573 2024-11-10T20:25:43,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40573 2024-11-10T20:25:43,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40573 2024-11-10T20:25:43,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40573 2024-11-10T20:25:43,170 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40573 2024-11-10T20:25:43,185 DEBUG [M:0;734d44df782b:45501 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;734d44df782b:45501 2024-11-10T20:25:43,185 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/734d44df782b,45501,1731270342389 2024-11-10T20:25:43,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:25:43,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:25:43,194 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/734d44df782b,45501,1731270342389 2024-11-10T20:25:43,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T20:25:43,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,216 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T20:25:43,217 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/734d44df782b,45501,1731270342389 from backup master directory 2024-11-10T20:25:43,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/734d44df782b,45501,1731270342389 2024-11-10T20:25:43,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:25:43,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:25:43,220 WARN [master/734d44df782b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:25:43,221 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=734d44df782b,45501,1731270342389 2024-11-10T20:25:43,223 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T20:25:43,224 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T20:25:43,285 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/hbase.id] with ID: 3d347b85-72dc-4dbf-9d56-a03cc614c94c 2024-11-10T20:25:43,285 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/.tmp/hbase.id 2024-11-10T20:25:43,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:25:43,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:25:43,299 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/.tmp/hbase.id]:[hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/hbase.id] 2024-11-10T20:25:43,342 INFO [master/734d44df782b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:25:43,346 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T20:25:43,365 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-10T20:25:43,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:25:43,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:25:43,401 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:25:43,404 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T20:25:43,411 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:25:43,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:25:43,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:25:43,462 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store 2024-11-10T20:25:43,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:25:43,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:25:43,487 INFO [master/734d44df782b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T20:25:43,491 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:25:43,492 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:25:43,492 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:25:43,492 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:25:43,494 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:25:43,494 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:25:43,494 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:25:43,495 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270343492Disabling compacts and flushes for region at 1731270343492Disabling writes for close at 1731270343494 (+2 ms)Writing region close event to WAL at 1731270343494Closed at 1731270343494 2024-11-10T20:25:43,497 WARN [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/.initializing 2024-11-10T20:25:43,498 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/WALs/734d44df782b,45501,1731270342389 2024-11-10T20:25:43,521 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C45501%2C1731270342389, suffix=, logDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/WALs/734d44df782b,45501,1731270342389, archiveDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/oldWALs, maxLogs=10 2024-11-10T20:25:43,531 INFO [master/734d44df782b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C45501%2C1731270342389.1731270343527 2024-11-10T20:25:43,551 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/WALs/734d44df782b,45501,1731270342389/734d44df782b%2C45501%2C1731270342389.1731270343527 2024-11-10T20:25:43,560 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45365:45365),(127.0.0.1/127.0.0.1:40219:40219)] 2024-11-10T20:25:43,562 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:25:43,562 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:25:43,566 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,567 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,605 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T20:25:43,636 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:43,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:25:43,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T20:25:43,642 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:43,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:25:43,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T20:25:43,647 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:43,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:25:43,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T20:25:43,651 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:43,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:25:43,652 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,656 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,657 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,662 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,663 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,666 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T20:25:43,670 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:25:43,674 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:25:43,675 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=737758, jitterRate=-0.061893001198768616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T20:25:43,681 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731270343580Initializing all the Stores at 1731270343582 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270343583 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270343584 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270343584Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270343584Cleaning up temporary data from old regions at 1731270343663 (+79 ms)Region opened successfully at 1731270343681 (+18 ms) 2024-11-10T20:25:43,682 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T20:25:43,716 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265997b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:25:43,748 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T20:25:43,759 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T20:25:43,760 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T20:25:43,763 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T20:25:43,764 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T20:25:43,769 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-10T20:25:43,769 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T20:25:43,794 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T20:25:43,802 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T20:25:43,804 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T20:25:43,806 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T20:25:43,807 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T20:25:43,809 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T20:25:43,811 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T20:25:43,814 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T20:25:43,816 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T20:25:43,817 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T20:25:43,819 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T20:25:43,835 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T20:25:43,836 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T20:25:43,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:25:43,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:25:43,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,843 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=734d44df782b,45501,1731270342389, sessionid=0x10193dc68fd0000, setting cluster-up flag (Was=false) 2024-11-10T20:25:43,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,862 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T20:25:43,864 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,45501,1731270342389 2024-11-10T20:25:43,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:43,875 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T20:25:43,877 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,45501,1731270342389 2024-11-10T20:25:43,883 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T20:25:43,955 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T20:25:43,966 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T20:25:43,973 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T20:25:43,974 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(746): ClusterId : 3d347b85-72dc-4dbf-9d56-a03cc614c94c 2024-11-10T20:25:43,976 DEBUG [RS:0;734d44df782b:40573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T20:25:43,981 DEBUG [RS:0;734d44df782b:40573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T20:25:43,981 DEBUG [RS:0;734d44df782b:40573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T20:25:43,980 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 734d44df782b,45501,1731270342389 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T20:25:43,985 DEBUG [RS:0;734d44df782b:40573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T20:25:43,985 DEBUG [RS:0;734d44df782b:40573 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6257bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:25:43,987 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:25:43,987 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:25:43,987 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:25:43,987 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:25:43,988 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/734d44df782b:0, corePoolSize=10, maxPoolSize=10 2024-11-10T20:25:43,988 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:43,988 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:25:43,988 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:43,994 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731270373994 2024-11-10T20:25:43,994 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:25:43,995 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T20:25:43,995 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T20:25:43,996 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T20:25:44,001 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T20:25:44,001 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T20:25:44,002 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T20:25:44,002 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T20:25:44,002 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:44,002 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T20:25:44,002 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,006 DEBUG [RS:0;734d44df782b:40573 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;734d44df782b:40573 2024-11-10T20:25:44,009 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T20:25:44,010 INFO [RS:0;734d44df782b:40573 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T20:25:44,010 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T20:25:44,011 INFO [RS:0;734d44df782b:40573 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T20:25:44,011 DEBUG [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T20:25:44,011 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T20:25:44,014 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(2659): reportForDuty to master=734d44df782b,45501,1731270342389 with port=40573, startcode=1731270343080 2024-11-10T20:25:44,015 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T20:25:44,015 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T20:25:44,017 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270344017,5,FailOnTimeoutGroup] 2024-11-10T20:25:44,018 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270344018,5,FailOnTimeoutGroup] 2024-11-10T20:25:44,018 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,018 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T20:25:44,020 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,020 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:25:44,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:25:44,024 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T20:25:44,025 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b 2024-11-10T20:25:44,028 DEBUG [RS:0;734d44df782b:40573 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T20:25:44,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:25:44,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:25:44,041 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:25:44,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:25:44,047 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:25:44,047 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:44,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:25:44,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:25:44,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:25:44,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:44,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:25:44,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:25:44,055 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:25:44,055 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:44,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:25:44,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:25:44,058 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:25:44,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:44,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:25:44,060 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:25:44,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740 2024-11-10T20:25:44,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740 2024-11-10T20:25:44,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:25:44,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:25:44,067 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:25:44,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:25:44,075 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:25:44,076 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=772702, jitterRate=-0.017459675669670105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:25:44,082 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731270344041Initializing all the Stores at 1731270344043 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270344043Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270344043Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270344043Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270344044 (+1 ms)Cleaning up temporary data from old regions at 1731270344065 (+21 ms)Region opened successfully at 1731270344081 (+16 ms) 2024-11-10T20:25:44,082 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:25:44,082 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:25:44,082 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:25:44,082 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:25:44,082 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:25:44,084 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:25:44,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270344082Disabling compacts and flushes for region at 1731270344082Disabling writes for close at 1731270344082Writing region close event to WAL at 1731270344083 (+1 ms)Closed at 1731270344084 (+1 ms) 2024-11-10T20:25:44,088 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:25:44,088 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T20:25:44,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T20:25:44,109 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:25:44,114 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T20:25:44,114 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45493, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T20:25:44,121 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45501 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 734d44df782b,40573,1731270343080 2024-11-10T20:25:44,123 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45501 {}] master.ServerManager(517): Registering regionserver=734d44df782b,40573,1731270343080 2024-11-10T20:25:44,138 DEBUG [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b 2024-11-10T20:25:44,139 DEBUG [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41563 2024-11-10T20:25:44,139 DEBUG [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T20:25:44,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:25:44,144 DEBUG [RS:0;734d44df782b:40573 {}] zookeeper.ZKUtil(111): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/734d44df782b,40573,1731270343080 2024-11-10T20:25:44,144 WARN [RS:0;734d44df782b:40573 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:25:44,144 INFO [RS:0;734d44df782b:40573 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:25:44,145 DEBUG [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080 2024-11-10T20:25:44,146 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [734d44df782b,40573,1731270343080] 2024-11-10T20:25:44,172 INFO [RS:0;734d44df782b:40573 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T20:25:44,185 INFO [RS:0;734d44df782b:40573 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T20:25:44,190 INFO [RS:0;734d44df782b:40573 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T20:25:44,190 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,192 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T20:25:44,198 INFO [RS:0;734d44df782b:40573 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T20:25:44,199 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,200 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,200 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,200 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,200 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,200 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,200 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:25:44,201 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,201 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,201 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,201 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,201 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,201 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:25:44,201 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:25:44,201 DEBUG [RS:0;734d44df782b:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:25:44,202 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,203 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,203 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,203 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,203 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,204 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,40573,1731270343080-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:25:44,223 INFO [RS:0;734d44df782b:40573 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T20:25:44,225 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,40573,1731270343080-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,226 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,226 INFO [RS:0;734d44df782b:40573 {}] regionserver.Replication(171): 734d44df782b,40573,1731270343080 started 2024-11-10T20:25:44,245 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,246 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(1482): Serving as 734d44df782b,40573,1731270343080, RpcServer on 734d44df782b/172.17.0.2:40573, sessionid=0x10193dc68fd0001 2024-11-10T20:25:44,247 DEBUG [RS:0;734d44df782b:40573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T20:25:44,247 DEBUG [RS:0;734d44df782b:40573 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 734d44df782b,40573,1731270343080 2024-11-10T20:25:44,247 DEBUG [RS:0;734d44df782b:40573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,40573,1731270343080' 2024-11-10T20:25:44,247 DEBUG [RS:0;734d44df782b:40573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T20:25:44,248 DEBUG [RS:0;734d44df782b:40573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T20:25:44,249 DEBUG [RS:0;734d44df782b:40573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T20:25:44,249 DEBUG [RS:0;734d44df782b:40573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T20:25:44,249 DEBUG [RS:0;734d44df782b:40573 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 734d44df782b,40573,1731270343080 2024-11-10T20:25:44,250 DEBUG [RS:0;734d44df782b:40573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,40573,1731270343080' 2024-11-10T20:25:44,250 DEBUG [RS:0;734d44df782b:40573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T20:25:44,250 DEBUG [RS:0;734d44df782b:40573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T20:25:44,251 DEBUG [RS:0;734d44df782b:40573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T20:25:44,251 INFO [RS:0;734d44df782b:40573 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T20:25:44,251 INFO [RS:0;734d44df782b:40573 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T20:25:44,265 WARN [734d44df782b:45501 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T20:25:44,360 INFO [RS:0;734d44df782b:40573 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C40573%2C1731270343080, suffix=, logDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080, archiveDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs, maxLogs=32 2024-11-10T20:25:44,363 INFO [RS:0;734d44df782b:40573 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.1731270344362 2024-11-10T20:25:44,372 INFO [RS:0;734d44df782b:40573 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270344362 2024-11-10T20:25:44,374 DEBUG [RS:0;734d44df782b:40573 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40219:40219),(127.0.0.1/127.0.0.1:45365:45365)] 2024-11-10T20:25:44,517 DEBUG [734d44df782b:45501 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T20:25:44,529 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=734d44df782b,40573,1731270343080 2024-11-10T20:25:44,536 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,40573,1731270343080, state=OPENING 2024-11-10T20:25:44,542 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T20:25:44,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:44,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:25:44,545 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:25:44,545 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:25:44,546 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:25:44,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,40573,1731270343080}] 2024-11-10T20:25:44,725 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T20:25:44,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37405, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T20:25:44,738 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T20:25:44,739 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:25:44,742 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C40573%2C1731270343080.meta, suffix=.meta, logDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080, archiveDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs, maxLogs=32 2024-11-10T20:25:44,744 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.meta.1731270344743.meta 2024-11-10T20:25:44,751 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.meta.1731270344743.meta 2024-11-10T20:25:44,753 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45365:45365),(127.0.0.1/127.0.0.1:40219:40219)] 2024-11-10T20:25:44,756 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:25:44,758 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T20:25:44,760 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T20:25:44,766 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T20:25:44,770 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T20:25:44,771 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:25:44,771 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T20:25:44,771 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T20:25:44,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:25:44,776 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:25:44,776 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:44,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:25:44,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:25:44,779 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:25:44,779 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:44,780 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:25:44,780 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:25:44,781 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:25:44,781 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:44,782 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:25:44,782 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:25:44,783 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:25:44,783 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:44,784 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:25:44,784 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:25:44,785 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740 2024-11-10T20:25:44,788 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740 2024-11-10T20:25:44,790 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:25:44,790 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:25:44,791 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:25:44,794 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:25:44,795 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=701402, jitterRate=-0.10812202095985413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:25:44,796 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T20:25:44,797 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731270344772Writing region info on filesystem at 1731270344772Initializing all the Stores at 1731270344774 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270344774Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270344774Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270344774Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270344774Cleaning up temporary data from old regions at 1731270344790 (+16 ms)Running coprocessor post-open hooks at 1731270344796 (+6 ms)Region opened successfully at 1731270344797 (+1 ms) 2024-11-10T20:25:44,804 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731270344715 2024-11-10T20:25:44,815 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T20:25:44,816 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T20:25:44,818 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,40573,1731270343080 2024-11-10T20:25:44,819 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,40573,1731270343080, state=OPEN 2024-11-10T20:25:44,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:25:44,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:25:44,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:25:44,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:25:44,824 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=734d44df782b,40573,1731270343080 2024-11-10T20:25:44,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T20:25:44,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,40573,1731270343080 in 277 msec 2024-11-10T20:25:44,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T20:25:44,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 735 msec 2024-11-10T20:25:44,839 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:25:44,839 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T20:25:44,859 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:25:44,861 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,40573,1731270343080, seqNum=-1] 2024-11-10T20:25:44,882 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:25:44,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57115, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:25:44,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 994 msec 2024-11-10T20:25:44,905 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731270344905, completionTime=-1 2024-11-10T20:25:44,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T20:25:44,908 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T20:25:44,935 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T20:25:44,935 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731270404935 2024-11-10T20:25:44,935 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731270464935 2024-11-10T20:25:44,935 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-10T20:25:44,938 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,45501,1731270342389-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,938 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,45501,1731270342389-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,938 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,45501,1731270342389-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,940 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-734d44df782b:45501, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,940 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,941 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:44,946 DEBUG [master/734d44df782b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T20:25:44,968 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.746sec 2024-11-10T20:25:44,969 INFO [master/734d44df782b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T20:25:44,971 INFO [master/734d44df782b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T20:25:44,972 INFO [master/734d44df782b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T20:25:44,972 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T20:25:44,972 INFO [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T20:25:44,973 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,45501,1731270342389-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:25:44,974 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,45501,1731270342389-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T20:25:44,983 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T20:25:44,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b598c24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:25:44,984 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T20:25:44,985 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,45501,1731270342389-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:25:45,006 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T20:25:45,007 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T20:25:45,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 734d44df782b,45501,-1 for getting cluster id 2024-11-10T20:25:45,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T20:25:45,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3d347b85-72dc-4dbf-9d56-a03cc614c94c' 2024-11-10T20:25:45,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T20:25:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3d347b85-72dc-4dbf-9d56-a03cc614c94c" 2024-11-10T20:25:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd1d57c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:25:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [734d44df782b,45501,-1] 2024-11-10T20:25:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T20:25:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:25:45,042 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T20:25:45,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e15a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:25:45,047 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:25:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,40573,1731270343080, seqNum=-1] 2024-11-10T20:25:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:25:45,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:25:45,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=734d44df782b,45501,1731270342389 2024-11-10T20:25:45,081 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:25:45,089 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T20:25:45,093 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T20:25:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 734d44df782b,45501,1731270342389 2024-11-10T20:25:45,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53459839 2024-11-10T20:25:45,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T20:25:45,105 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T20:25:45,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45501 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T20:25:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45501 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T20:25:45,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45501 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:25:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45501 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-11-10T20:25:45,122 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T20:25:45,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45501 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-11-10T20:25:45,125 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:45,127 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T20:25:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:25:45,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741835_1011 (size=389) 2024-11-10T20:25:45,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741835_1011 (size=389) 2024-11-10T20:25:45,175 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c5a882dd0964dc3f7a89dc6e7bcf7b2, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b 2024-11-10T20:25:45,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741836_1012 (size=72) 2024-11-10T20:25:45,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741836_1012 (size=72) 2024-11-10T20:25:45,188 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:25:45,188 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 4c5a882dd0964dc3f7a89dc6e7bcf7b2, disabling compactions & flushes 2024-11-10T20:25:45,188 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:25:45,188 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:25:45,188 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. after waiting 0 ms 2024-11-10T20:25:45,188 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:25:45,188 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:25:45,188 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c5a882dd0964dc3f7a89dc6e7bcf7b2: Waiting for close lock at 1731270345188Disabling compacts and flushes for region at 1731270345188Disabling writes for close at 1731270345188Writing region close event to WAL at 1731270345188Closed at 1731270345188 2024-11-10T20:25:45,191 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T20:25:45,196 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731270345191"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731270345191"}]},"ts":"1731270345191"} 2024-11-10T20:25:45,202 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T20:25:45,204 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T20:25:45,206 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270345204"}]},"ts":"1731270345204"} 2024-11-10T20:25:45,211 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-11-10T20:25:45,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=4c5a882dd0964dc3f7a89dc6e7bcf7b2, ASSIGN}] 2024-11-10T20:25:45,216 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=4c5a882dd0964dc3f7a89dc6e7bcf7b2, ASSIGN 2024-11-10T20:25:45,218 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=4c5a882dd0964dc3f7a89dc6e7bcf7b2, ASSIGN; state=OFFLINE, location=734d44df782b,40573,1731270343080; forceNewPlan=false, retain=false 2024-11-10T20:25:45,369 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c5a882dd0964dc3f7a89dc6e7bcf7b2, regionState=OPENING, regionLocation=734d44df782b,40573,1731270343080 2024-11-10T20:25:45,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=4c5a882dd0964dc3f7a89dc6e7bcf7b2, ASSIGN because future has completed 2024-11-10T20:25:45,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c5a882dd0964dc3f7a89dc6e7bcf7b2, server=734d44df782b,40573,1731270343080}] 2024-11-10T20:25:45,537 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:25:45,537 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c5a882dd0964dc3f7a89dc6e7bcf7b2, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:25:45,538 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,538 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:25:45,538 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,538 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,540 INFO [StoreOpener-4c5a882dd0964dc3f7a89dc6e7bcf7b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,543 INFO [StoreOpener-4c5a882dd0964dc3f7a89dc6e7bcf7b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c5a882dd0964dc3f7a89dc6e7bcf7b2 columnFamilyName info 2024-11-10T20:25:45,543 DEBUG [StoreOpener-4c5a882dd0964dc3f7a89dc6e7bcf7b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:25:45,544 INFO [StoreOpener-4c5a882dd0964dc3f7a89dc6e7bcf7b2-1 {}] regionserver.HStore(327): Store=4c5a882dd0964dc3f7a89dc6e7bcf7b2/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:25:45,544 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,545 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,546 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,547 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,547 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,550 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,553 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:25:45,554 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4c5a882dd0964dc3f7a89dc6e7bcf7b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=844965, jitterRate=0.07442891597747803}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T20:25:45,554 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:25:45,555 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4c5a882dd0964dc3f7a89dc6e7bcf7b2: Running coprocessor pre-open hook at 1731270345538Writing region info on filesystem at 1731270345538Initializing all the Stores at 1731270345540 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270345540Cleaning up temporary data from old regions at 1731270345547 (+7 ms)Running coprocessor post-open hooks at 1731270345554 (+7 ms)Region opened successfully at 1731270345555 (+1 ms) 2024-11-10T20:25:45,557 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2., pid=6, masterSystemTime=1731270345530 2024-11-10T20:25:45,561 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:25:45,561 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:25:45,562 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c5a882dd0964dc3f7a89dc6e7bcf7b2, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,40573,1731270343080 2024-11-10T20:25:45,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c5a882dd0964dc3f7a89dc6e7bcf7b2, server=734d44df782b,40573,1731270343080 because future has completed 2024-11-10T20:25:45,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T20:25:45,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c5a882dd0964dc3f7a89dc6e7bcf7b2, server=734d44df782b,40573,1731270343080 in 193 msec 2024-11-10T20:25:45,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T20:25:45,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=4c5a882dd0964dc3f7a89dc6e7bcf7b2, ASSIGN in 359 msec 2024-11-10T20:25:45,578 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T20:25:45,578 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270345578"}]},"ts":"1731270345578"} 2024-11-10T20:25:45,582 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-11-10T20:25:45,583 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T20:25:45,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 469 msec 2024-11-10T20:25:50,274 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T20:25:50,318 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T20:25:50,319 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-11-10T20:25:52,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T20:25:52,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T20:25:52,807 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-10T20:25:52,807 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-10T20:25:52,808 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:25:52,808 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T20:25:52,808 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T20:25:52,808 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T20:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45501 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:25:55,143 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-11-10T20:25:55,146 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-11-10T20:25:55,152 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-11-10T20:25:55,153 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:25:55,153 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.1731270355153 2024-11-10T20:25:55,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:25:55,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:25:55,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:25:55,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:25:55,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:25:55,163 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270344362 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270355153 2024-11-10T20:25:55,166 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40219:40219),(127.0.0.1/127.0.0.1:45365:45365)] 2024-11-10T20:25:55,166 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270344362 is not closed yet, will try archiving it next time 2024-11-10T20:25:55,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741833_1009 (size=451) 2024-11-10T20:25:55,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741833_1009 (size=451) 2024-11-10T20:25:55,172 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270344362 to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs/734d44df782b%2C40573%2C1731270343080.1731270344362 2024-11-10T20:25:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2., hostname=734d44df782b,40573,1731270343080, seqNum=2] 2024-11-10T20:26:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40573 {}] regionserver.HRegion(8855): Flush requested on 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:26:07,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c5a882dd0964dc3f7a89dc6e7bcf7b2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:26:07,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/e4d75004cae14267bfccd05653428ad3 is 1080, key is row0001/info:/1731270355178/Put/seqid=0 2024-11-10T20:26:07,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741838_1014 (size=12509) 2024-11-10T20:26:07,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741838_1014 (size=12509) 2024-11-10T20:26:07,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/e4d75004cae14267bfccd05653428ad3 2024-11-10T20:26:07,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/e4d75004cae14267bfccd05653428ad3 as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/e4d75004cae14267bfccd05653428ad3 2024-11-10T20:26:07,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/e4d75004cae14267bfccd05653428ad3, entries=7, sequenceid=11, filesize=12.2 K 2024-11-10T20:26:07,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 in 149ms, sequenceid=11, compaction requested=false 2024-11-10T20:26:07,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c5a882dd0964dc3f7a89dc6e7bcf7b2: 2024-11-10T20:26:11,464 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T20:26:15,226 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.1731270375226 2024-11-10T20:26:15,435 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK], DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK]] 2024-11-10T20:26:15,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:15,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:15,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:15,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:15,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:15,436 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270355153 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270375226 2024-11-10T20:26:15,437 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45365:45365),(127.0.0.1/127.0.0.1:40219:40219)] 2024-11-10T20:26:15,437 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270355153 is not closed yet, will try archiving it next time 2024-11-10T20:26:15,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741837_1013 (size=12399) 2024-11-10T20:26:15,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741837_1013 (size=12399) 2024-11-10T20:26:15,641 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:17,845 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:20,049 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:22,253 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40573 {}] regionserver.HRegion(8855): Flush requested on 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:26:22,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c5a882dd0964dc3f7a89dc6e7bcf7b2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:26:22,456 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:22,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/8a33b712d56641daa4190f5d0c6b0e6a is 1080, key is row0008/info:/1731270369211/Put/seqid=0 2024-11-10T20:26:22,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741840_1016 (size=12509) 2024-11-10T20:26:22,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741840_1016 (size=12509) 2024-11-10T20:26:22,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/8a33b712d56641daa4190f5d0c6b0e6a 2024-11-10T20:26:22,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/8a33b712d56641daa4190f5d0c6b0e6a as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/8a33b712d56641daa4190f5d0c6b0e6a 2024-11-10T20:26:22,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/8a33b712d56641daa4190f5d0c6b0e6a, entries=7, sequenceid=21, filesize=12.2 K 2024-11-10T20:26:22,693 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:22,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 in 440ms, sequenceid=21, compaction requested=false 2024-11-10T20:26:22,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c5a882dd0964dc3f7a89dc6e7bcf7b2: 2024-11-10T20:26:22,693 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K 2024-11-10T20:26:22,693 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:26:22,694 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/e4d75004cae14267bfccd05653428ad3 because midkey is the same as first or last row 2024-11-10T20:26:24,458 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:24,986 INFO [master/734d44df782b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T20:26:24,986 INFO [master/734d44df782b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T20:26:26,661 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:26,663 WARN [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:26,664 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C40573%2C1731270343080:(num 1731270375226) roll requested 2024-11-10T20:26:26,665 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.1731270386665 2024-11-10T20:26:26,873 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:26,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:26,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:26,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:26,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:26,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:26,875 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270375226 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270386665 2024-11-10T20:26:26,876 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45365:45365),(127.0.0.1/127.0.0.1:40219:40219)] 2024-11-10T20:26:26,876 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270375226 is not closed yet, will try archiving it next time 2024-11-10T20:26:26,876 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270355153 to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs/734d44df782b%2C40573%2C1731270343080.1731270355153 2024-11-10T20:26:26,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741839_1015 (size=7739) 2024-11-10T20:26:26,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741839_1015 (size=7739) 2024-11-10T20:26:28,866 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:30,538 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 4c5a882dd0964dc3f7a89dc6e7bcf7b2, had cached 0 bytes from a total of 25018 2024-11-10T20:26:31,070 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:33,274 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:35,478 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:37,480 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T20:26:37,481 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.1731270397481 2024-11-10T20:26:41,465 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T20:26:42,489 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:42,492 WARN [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:42,492 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C40573%2C1731270343080:(num 1731270397481) roll requested 2024-11-10T20:26:42,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:42,493 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:42,493 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:42,493 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:42,493 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:42,494 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270386665 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270397481 2024-11-10T20:26:42,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45365:45365),(127.0.0.1/127.0.0.1:40219:40219)] 2024-11-10T20:26:42,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270386665 is not closed yet, will try archiving it next time 2024-11-10T20:26:42,495 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.1731270402495 2024-11-10T20:26:42,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741841_1017 (size=4753) 2024-11-10T20:26:42,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741841_1017 (size=4753) 2024-11-10T20:26:47,499 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:47,499 WARN [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40573 {}] regionserver.HRegion(8855): Flush requested on 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:26:47,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c5a882dd0964dc3f7a89dc6e7bcf7b2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:26:47,506 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:47,506 WARN [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:49,500 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T20:26:52,501 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:52,502 WARN [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK], DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK]] 2024-11-10T20:26:52,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:52,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:52,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:52,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:52,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:52,503 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270397481 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270402495 2024-11-10T20:26:52,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741842_1018 (size=1569) 2024-11-10T20:26:52,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741842_1018 (size=1569) 2024-11-10T20:26:52,507 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40219:40219),(127.0.0.1/127.0.0.1:45365:45365)] 2024-11-10T20:26:52,507 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270397481 is not closed yet, will try archiving it next time 2024-11-10T20:26:52,507 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C40573%2C1731270343080:(num 1731270402495) roll requested 2024-11-10T20:26:52,507 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.1731270412507 2024-11-10T20:26:52,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/b8bdaabf5b4244bea82eb60b5876468f is 1080, key is row0015/info:/1731270384256/Put/seqid=0 2024-11-10T20:26:52,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741844_1020 (size=12509) 2024-11-10T20:26:52,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741844_1020 (size=12509) 2024-11-10T20:26:52,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/b8bdaabf5b4244bea82eb60b5876468f 2024-11-10T20:26:52,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/b8bdaabf5b4244bea82eb60b5876468f as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/b8bdaabf5b4244bea82eb60b5876468f 2024-11-10T20:26:52,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/b8bdaabf5b4244bea82eb60b5876468f, entries=7, sequenceid=31, filesize=12.2 K 2024-11-10T20:26:57,515 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK], DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK]] 2024-11-10T20:26:57,515 WARN [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK], DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK]] 2024-11-10T20:26:57,537 INFO [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK], DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK]] 2024-11-10T20:26:57,538 WARN [FSHLog-0-hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b-prefix:734d44df782b,40573,1731270343080 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41181,DS-222a65c5-269a-469b-90f5-9489587483a7,DISK], DatanodeInfoWithStorage[127.0.0.1:33157,DS-fc1396c0-346a-48f1-a90f-c6b43d1398a4,DISK]] 2024-11-10T20:26:57,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 in 10038ms, sequenceid=31, compaction requested=true 2024-11-10T20:26:57,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c5a882dd0964dc3f7a89dc6e7bcf7b2: 2024-11-10T20:26:57,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K 2024-11-10T20:26:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:26:57,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/e4d75004cae14267bfccd05653428ad3 because midkey is the same as first or last row 2024-11-10T20:26:57,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,538 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270402495 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270412507 2024-11-10T20:26:57,539 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45365:45365),(127.0.0.1/127.0.0.1:40219:40219)] 2024-11-10T20:26:57,539 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270402495 is not closed yet, will try archiving it next time 2024-11-10T20:26:57,540 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270375226 to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs/734d44df782b%2C40573%2C1731270343080.1731270375226 2024-11-10T20:26:57,540 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C40573%2C1731270343080:(num 1731270412507) roll requested 2024-11-10T20:26:57,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c5a882dd0964dc3f7a89dc6e7bcf7b2:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:26:57,540 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.1731270417540 2024-11-10T20:26:57,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741843_1019 (size=438) 2024-11-10T20:26:57,543 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270386665 to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs/734d44df782b%2C40573%2C1731270343080.1731270386665 2024-11-10T20:26:57,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741843_1019 (size=438) 2024-11-10T20:26:57,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:26:57,544 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:26:57,544 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270397481 to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs/734d44df782b%2C40573%2C1731270343080.1731270397481 2024-11-10T20:26:57,546 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270402495 to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs/734d44df782b%2C40573%2C1731270343080.1731270402495 2024-11-10T20:26:57,546 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:26:57,548 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.HStore(1541): 4c5a882dd0964dc3f7a89dc6e7bcf7b2/info is initiating minor compaction (all files) 2024-11-10T20:26:57,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,549 INFO [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c5a882dd0964dc3f7a89dc6e7bcf7b2/info in TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:26:57,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,549 INFO [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/e4d75004cae14267bfccd05653428ad3, hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/8a33b712d56641daa4190f5d0c6b0e6a, hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/b8bdaabf5b4244bea82eb60b5876468f] into tmpdir=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp, totalSize=36.6 K 2024-11-10T20:26:57,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,549 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270412507 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270417540 2024-11-10T20:26:57,550 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45365:45365),(127.0.0.1/127.0.0.1:40219:40219)] 2024-11-10T20:26:57,550 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4d75004cae14267bfccd05653428ad3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731270355178 2024-11-10T20:26:57,550 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270412507 is not closed yet, will try archiving it next time 2024-11-10T20:26:57,551 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40573%2C1731270343080.1731270417551 2024-11-10T20:26:57,551 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a33b712d56641daa4190f5d0c6b0e6a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731270369211 2024-11-10T20:26:57,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741845_1021 (size=93) 2024-11-10T20:26:57,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741845_1021 (size=93) 2024-11-10T20:26:57,553 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8bdaabf5b4244bea82eb60b5876468f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731270384256 2024-11-10T20:26:57,555 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270412507 to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs/734d44df782b%2C40573%2C1731270343080.1731270412507 2024-11-10T20:26:57,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:26:57,564 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270417540 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/WALs/734d44df782b,40573,1731270343080/734d44df782b%2C40573%2C1731270343080.1731270417551 2024-11-10T20:26:57,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741846_1022 (size=1258) 2024-11-10T20:26:57,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741846_1022 (size=1258) 2024-11-10T20:26:57,567 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40219:40219),(127.0.0.1/127.0.0.1:45365:45365)] 2024-11-10T20:26:57,583 INFO [RS:0;734d44df782b:40573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c5a882dd0964dc3f7a89dc6e7bcf7b2#info#compaction#3 average throughput is 21.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:26:57,584 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/fd7148bbc36549d1a2fd2b6f62c9a457 is 1080, key is row0001/info:/1731270355178/Put/seqid=0 2024-11-10T20:26:57,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741848_1024 (size=27710) 2024-11-10T20:26:57,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741848_1024 (size=27710) 2024-11-10T20:26:57,600 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/fd7148bbc36549d1a2fd2b6f62c9a457 as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/fd7148bbc36549d1a2fd2b6f62c9a457 2024-11-10T20:26:57,621 INFO [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c5a882dd0964dc3f7a89dc6e7bcf7b2/info of 4c5a882dd0964dc3f7a89dc6e7bcf7b2 into fd7148bbc36549d1a2fd2b6f62c9a457(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:26:57,621 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c5a882dd0964dc3f7a89dc6e7bcf7b2: 2024-11-10T20:26:57,624 INFO [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2., storeName=4c5a882dd0964dc3f7a89dc6e7bcf7b2/info, priority=13, startTime=1731270417539; duration=0sec 2024-11-10T20:26:57,624 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-10T20:26:57,624 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:26:57,624 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/fd7148bbc36549d1a2fd2b6f62c9a457 because midkey is the same as first or last row 2024-11-10T20:26:57,625 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-10T20:26:57,625 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:26:57,625 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/fd7148bbc36549d1a2fd2b6f62c9a457 because midkey is the same as first or last row 2024-11-10T20:26:57,625 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-10T20:26:57,625 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:26:57,625 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/fd7148bbc36549d1a2fd2b6f62c9a457 because midkey is the same as first or last row 2024-11-10T20:26:57,625 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:26:57,626 DEBUG [RS:0;734d44df782b:40573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c5a882dd0964dc3f7a89dc6e7bcf7b2:info 2024-11-10T20:27:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40573 {}] regionserver.HRegion(8855): Flush requested on 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:27:09,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c5a882dd0964dc3f7a89dc6e7bcf7b2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:27:09,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/058dda0f6f1946c28b41a38b4627e5fb is 1080, key is row0022/info:/1731270417552/Put/seqid=0 2024-11-10T20:27:09,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741849_1025 (size=12509) 2024-11-10T20:27:09,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741849_1025 (size=12509) 2024-11-10T20:27:09,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/058dda0f6f1946c28b41a38b4627e5fb 2024-11-10T20:27:09,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/058dda0f6f1946c28b41a38b4627e5fb as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/058dda0f6f1946c28b41a38b4627e5fb 2024-11-10T20:27:09,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/058dda0f6f1946c28b41a38b4627e5fb, entries=7, sequenceid=42, filesize=12.2 K 2024-11-10T20:27:09,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 in 35ms, sequenceid=42, compaction requested=false 2024-11-10T20:27:09,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c5a882dd0964dc3f7a89dc6e7bcf7b2: 2024-11-10T20:27:09,611 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K 2024-11-10T20:27:09,611 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:09,611 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/fd7148bbc36549d1a2fd2b6f62c9a457 because midkey is the same as first or last row 2024-11-10T20:27:11,465 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T20:27:15,538 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 4c5a882dd0964dc3f7a89dc6e7bcf7b2, had cached 0 bytes from a total of 40219 2024-11-10T20:27:17,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T20:27:17,587 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:27:17,587 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:17,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:17,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:17,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T20:27:17,593 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T20:27:17,593 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1075752841, stopped=false 2024-11-10T20:27:17,593 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=734d44df782b,45501,1731270342389 2024-11-10T20:27:17,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:17,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:17,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:17,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:27:17,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:17,596 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:27:17,596 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:17,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:17,596 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:17,596 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:17,596 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '734d44df782b,40573,1731270343080' ***** 2024-11-10T20:27:17,596 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T20:27:17,597 INFO [RS:0;734d44df782b:40573 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T20:27:17,597 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T20:27:17,597 INFO [RS:0;734d44df782b:40573 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T20:27:17,597 INFO [RS:0;734d44df782b:40573 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T20:27:17,598 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:27:17,598 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(959): stopping server 734d44df782b,40573,1731270343080 2024-11-10T20:27:17,598 INFO [RS:0;734d44df782b:40573 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:27:17,598 INFO [RS:0;734d44df782b:40573 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;734d44df782b:40573. 2024-11-10T20:27:17,598 DEBUG [RS:0;734d44df782b:40573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:17,598 DEBUG [RS:0;734d44df782b:40573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:17,599 INFO [RS:0;734d44df782b:40573 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T20:27:17,599 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4c5a882dd0964dc3f7a89dc6e7bcf7b2, disabling compactions & flushes 2024-11-10T20:27:17,599 INFO [RS:0;734d44df782b:40573 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T20:27:17,599 INFO [RS:0;734d44df782b:40573 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T20:27:17,599 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:27:17,599 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:27:17,599 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T20:27:17,599 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. after waiting 0 ms 2024-11-10T20:27:17,599 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:27:17,599 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4c5a882dd0964dc3f7a89dc6e7bcf7b2 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-11-10T20:27:17,599 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T20:27:17,599 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:27:17,599 DEBUG [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 4c5a882dd0964dc3f7a89dc6e7bcf7b2=TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.} 2024-11-10T20:27:17,599 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:27:17,599 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:27:17,600 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:27:17,600 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:27:17,600 DEBUG [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4c5a882dd0964dc3f7a89dc6e7bcf7b2 2024-11-10T20:27:17,600 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-11-10T20:27:17,606 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/dd4c226399344fd0ae45e6e72292396d is 1080, key is row0029/info:/1731270431577/Put/seqid=0 2024-11-10T20:27:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741850_1026 (size=8193) 2024-11-10T20:27:17,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741850_1026 (size=8193) 2024-11-10T20:27:17,616 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/dd4c226399344fd0ae45e6e72292396d 2024-11-10T20:27:17,623 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/.tmp/info/f7878186267c41dd843c7954dfeb2950 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2./info:regioninfo/1731270345562/Put/seqid=0 2024-11-10T20:27:17,627 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/.tmp/info/dd4c226399344fd0ae45e6e72292396d as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/dd4c226399344fd0ae45e6e72292396d 2024-11-10T20:27:17,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741851_1027 (size=7016) 2024-11-10T20:27:17,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741851_1027 (size=7016) 2024-11-10T20:27:17,630 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/.tmp/info/f7878186267c41dd843c7954dfeb2950 2024-11-10T20:27:17,638 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/dd4c226399344fd0ae45e6e72292396d, entries=3, sequenceid=48, filesize=8.0 K 2024-11-10T20:27:17,639 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 in 40ms, sequenceid=48, compaction requested=true 2024-11-10T20:27:17,640 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/e4d75004cae14267bfccd05653428ad3, hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/8a33b712d56641daa4190f5d0c6b0e6a, hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/b8bdaabf5b4244bea82eb60b5876468f] to archive 2024-11-10T20:27:17,643 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T20:27:17,646 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/e4d75004cae14267bfccd05653428ad3 to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/e4d75004cae14267bfccd05653428ad3 2024-11-10T20:27:17,648 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/8a33b712d56641daa4190f5d0c6b0e6a to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/8a33b712d56641daa4190f5d0c6b0e6a 2024-11-10T20:27:17,650 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/b8bdaabf5b4244bea82eb60b5876468f to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/info/b8bdaabf5b4244bea82eb60b5876468f 2024-11-10T20:27:17,662 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/.tmp/ns/01a8e141d1fd4b5baa0be3cd70a045c5 is 43, key is default/ns:d/1731270344889/Put/seqid=0 2024-11-10T20:27:17,663 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=734d44df782b:45501 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-10T20:27:17,668 WARN [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [e4d75004cae14267bfccd05653428ad3=12509, 8a33b712d56641daa4190f5d0c6b0e6a=12509, b8bdaabf5b4244bea82eb60b5876468f=12509] 2024-11-10T20:27:17,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741852_1028 (size=5153) 2024-11-10T20:27:17,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741852_1028 (size=5153) 2024-11-10T20:27:17,670 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/.tmp/ns/01a8e141d1fd4b5baa0be3cd70a045c5 2024-11-10T20:27:17,675 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/default/TestLogRolling-testSlowSyncLogRolling/4c5a882dd0964dc3f7a89dc6e7bcf7b2/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-11-10T20:27:17,678 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:27:17,678 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4c5a882dd0964dc3f7a89dc6e7bcf7b2: Waiting for close lock at 1731270437598Running coprocessor pre-close hooks at 1731270437599 (+1 ms)Disabling compacts and flushes for region at 1731270437599Disabling writes for close at 1731270437599Obtaining lock to block concurrent updates at 1731270437599Preparing flush snapshotting stores in 4c5a882dd0964dc3f7a89dc6e7bcf7b2 at 1731270437599Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731270437599Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. at 1731270437601 (+2 ms)Flushing 4c5a882dd0964dc3f7a89dc6e7bcf7b2/info: creating writer at 1731270437601Flushing 4c5a882dd0964dc3f7a89dc6e7bcf7b2/info: appending metadata at 1731270437605 (+4 ms)Flushing 4c5a882dd0964dc3f7a89dc6e7bcf7b2/info: closing flushed file at 1731270437605Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@747348d7: reopening flushed file at 1731270437626 (+21 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 4c5a882dd0964dc3f7a89dc6e7bcf7b2 in 40ms, sequenceid=48, compaction requested=true at 1731270437640 (+14 ms)Writing region close event to WAL at 1731270437670 (+30 ms)Running coprocessor post-close hooks at 1731270437676 (+6 ms)Closed at 1731270437678 (+2 ms) 2024-11-10T20:27:17,679 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731270345107.4c5a882dd0964dc3f7a89dc6e7bcf7b2. 2024-11-10T20:27:17,695 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/.tmp/table/d957952a9b9c4c4393682d0acabfd5b6 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731270345578/Put/seqid=0 2024-11-10T20:27:17,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741853_1029 (size=5396) 2024-11-10T20:27:17,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741853_1029 (size=5396) 2024-11-10T20:27:17,702 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/.tmp/table/d957952a9b9c4c4393682d0acabfd5b6 2024-11-10T20:27:17,712 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/.tmp/info/f7878186267c41dd843c7954dfeb2950 as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/info/f7878186267c41dd843c7954dfeb2950 2024-11-10T20:27:17,721 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/info/f7878186267c41dd843c7954dfeb2950, entries=10, sequenceid=11, filesize=6.9 K 2024-11-10T20:27:17,723 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/.tmp/ns/01a8e141d1fd4b5baa0be3cd70a045c5 as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/ns/01a8e141d1fd4b5baa0be3cd70a045c5 2024-11-10T20:27:17,732 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/ns/01a8e141d1fd4b5baa0be3cd70a045c5, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T20:27:17,733 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/.tmp/table/d957952a9b9c4c4393682d0acabfd5b6 as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/table/d957952a9b9c4c4393682d0acabfd5b6 2024-11-10T20:27:17,742 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/table/d957952a9b9c4c4393682d0acabfd5b6, entries=2, sequenceid=11, filesize=5.3 K 2024-11-10T20:27:17,744 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 144ms, sequenceid=11, compaction requested=false 2024-11-10T20:27:17,751 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T20:27:17,752 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:27:17,752 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:27:17,752 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270437599Running coprocessor pre-close hooks at 1731270437599Disabling compacts and flushes for region at 1731270437599Disabling writes for close at 1731270437600 (+1 ms)Obtaining lock to block concurrent updates at 1731270437600Preparing flush snapshotting stores in 1588230740 at 1731270437600Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731270437600Flushing stores of hbase:meta,,1.1588230740 at 1731270437601 (+1 ms)Flushing 1588230740/info: creating writer at 1731270437601Flushing 1588230740/info: appending metadata at 1731270437622 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731270437622Flushing 1588230740/ns: creating writer at 1731270437639 (+17 ms)Flushing 1588230740/ns: appending metadata at 1731270437661 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731270437661Flushing 1588230740/table: creating writer at 1731270437679 (+18 ms)Flushing 1588230740/table: appending metadata at 1731270437695 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731270437695Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b9e44b4: reopening flushed file at 1731270437711 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b040d63: reopening flushed file at 1731270437722 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a9fd267: reopening flushed file at 1731270437732 (+10 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 144ms, sequenceid=11, compaction requested=false at 1731270437744 (+12 ms)Writing region close event to WAL at 1731270437746 (+2 ms)Running coprocessor post-close hooks at 1731270437752 (+6 ms)Closed at 1731270437752 2024-11-10T20:27:17,753 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T20:27:17,800 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(976): stopping server 734d44df782b,40573,1731270343080; all regions closed. 2024-11-10T20:27:17,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741834_1010 (size=3066) 2024-11-10T20:27:17,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741834_1010 (size=3066) 2024-11-10T20:27:17,808 DEBUG [RS:0;734d44df782b:40573 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs 2024-11-10T20:27:17,808 INFO [RS:0;734d44df782b:40573 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C40573%2C1731270343080.meta:.meta(num 1731270344743) 2024-11-10T20:27:17,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:17,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741847_1023 (size=12695) 2024-11-10T20:27:17,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741847_1023 (size=12695) 2024-11-10T20:27:17,815 DEBUG [RS:0;734d44df782b:40573 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/oldWALs 2024-11-10T20:27:17,815 INFO [RS:0;734d44df782b:40573 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C40573%2C1731270343080:(num 1731270417551) 2024-11-10T20:27:17,816 DEBUG [RS:0;734d44df782b:40573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:17,816 INFO [RS:0;734d44df782b:40573 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:27:17,816 INFO [RS:0;734d44df782b:40573 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:27:17,816 INFO [RS:0;734d44df782b:40573 {}] hbase.ChoreService(370): Chore service for: regionserver/734d44df782b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T20:27:17,816 INFO [RS:0;734d44df782b:40573 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:27:17,816 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:27:17,817 INFO [RS:0;734d44df782b:40573 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40573 2024-11-10T20:27:17,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:27:17,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/734d44df782b,40573,1731270343080 2024-11-10T20:27:17,821 INFO [RS:0;734d44df782b:40573 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:27:17,823 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [734d44df782b,40573,1731270343080] 2024-11-10T20:27:17,824 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/734d44df782b,40573,1731270343080 already deleted, retry=false 2024-11-10T20:27:17,824 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 734d44df782b,40573,1731270343080 expired; onlineServers=0 2024-11-10T20:27:17,825 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '734d44df782b,45501,1731270342389' ***** 2024-11-10T20:27:17,825 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T20:27:17,825 INFO [M:0;734d44df782b:45501 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:27:17,825 INFO [M:0;734d44df782b:45501 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:27:17,825 DEBUG [M:0;734d44df782b:45501 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T20:27:17,825 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T20:27:17,825 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270344017 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270344017,5,FailOnTimeoutGroup] 2024-11-10T20:27:17,825 DEBUG [M:0;734d44df782b:45501 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T20:27:17,825 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270344018 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270344018,5,FailOnTimeoutGroup] 2024-11-10T20:27:17,826 INFO [M:0;734d44df782b:45501 {}] hbase.ChoreService(370): Chore service for: master/734d44df782b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T20:27:17,826 INFO [M:0;734d44df782b:45501 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:27:17,826 DEBUG [M:0;734d44df782b:45501 {}] master.HMaster(1795): Stopping service threads 2024-11-10T20:27:17,826 INFO [M:0;734d44df782b:45501 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T20:27:17,826 INFO [M:0;734d44df782b:45501 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:27:17,826 INFO [M:0;734d44df782b:45501 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T20:27:17,826 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T20:27:17,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T20:27:17,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:17,827 DEBUG [M:0;734d44df782b:45501 {}] zookeeper.ZKUtil(347): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T20:27:17,827 WARN [M:0;734d44df782b:45501 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T20:27:17,828 INFO [M:0;734d44df782b:45501 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/.lastflushedseqids 2024-11-10T20:27:17,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741854_1030 (size=130) 2024-11-10T20:27:17,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741854_1030 (size=130) 2024-11-10T20:27:17,842 INFO [M:0;734d44df782b:45501 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T20:27:17,842 INFO [M:0;734d44df782b:45501 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T20:27:17,842 DEBUG [M:0;734d44df782b:45501 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:27:17,842 INFO [M:0;734d44df782b:45501 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:17,842 DEBUG [M:0;734d44df782b:45501 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:17,842 DEBUG [M:0;734d44df782b:45501 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:27:17,842 DEBUG [M:0;734d44df782b:45501 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:17,843 INFO [M:0;734d44df782b:45501 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB 2024-11-10T20:27:17,861 DEBUG [M:0;734d44df782b:45501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3829e856bcd7409981a22461f2ea667c is 82, key is hbase:meta,,1/info:regioninfo/1731270344817/Put/seqid=0 2024-11-10T20:27:17,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741855_1031 (size=5672) 2024-11-10T20:27:17,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741855_1031 (size=5672) 2024-11-10T20:27:17,869 INFO [M:0;734d44df782b:45501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3829e856bcd7409981a22461f2ea667c 2024-11-10T20:27:17,892 DEBUG [M:0;734d44df782b:45501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2011e2e7ce446759e8332e83fbcd74b is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731270345585/Put/seqid=0 2024-11-10T20:27:17,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741856_1032 (size=6248) 2024-11-10T20:27:17,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741856_1032 (size=6248) 2024-11-10T20:27:17,899 INFO [M:0;734d44df782b:45501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2011e2e7ce446759e8332e83fbcd74b 2024-11-10T20:27:17,906 INFO [M:0;734d44df782b:45501 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2011e2e7ce446759e8332e83fbcd74b 2024-11-10T20:27:17,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:27:17,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10193dc68fd0001, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:27:17,923 INFO [RS:0;734d44df782b:40573 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:27:17,924 INFO [RS:0;734d44df782b:40573 {}] regionserver.HRegionServer(1031): Exiting; stopping=734d44df782b,40573,1731270343080; zookeeper connection closed. 2024-11-10T20:27:17,924 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a3a1de0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a3a1de0 2024-11-10T20:27:17,925 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T20:27:17,928 DEBUG [M:0;734d44df782b:45501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/59a591434100438ab4c914d99e273830 is 69, key is 734d44df782b,40573,1731270343080/rs:state/1731270344125/Put/seqid=0 2024-11-10T20:27:17,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741857_1033 (size=5156) 2024-11-10T20:27:17,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741857_1033 (size=5156) 2024-11-10T20:27:17,935 INFO [M:0;734d44df782b:45501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/59a591434100438ab4c914d99e273830 2024-11-10T20:27:17,959 DEBUG [M:0;734d44df782b:45501 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8b8369ae45184f92a4eec95f261e1489 is 52, key is load_balancer_on/state:d/1731270345086/Put/seqid=0 2024-11-10T20:27:17,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741858_1034 (size=5056) 2024-11-10T20:27:17,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741858_1034 (size=5056) 2024-11-10T20:27:17,967 INFO [M:0;734d44df782b:45501 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8b8369ae45184f92a4eec95f261e1489 2024-11-10T20:27:17,975 DEBUG [M:0;734d44df782b:45501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3829e856bcd7409981a22461f2ea667c as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3829e856bcd7409981a22461f2ea667c 2024-11-10T20:27:17,982 INFO [M:0;734d44df782b:45501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3829e856bcd7409981a22461f2ea667c, entries=8, sequenceid=59, filesize=5.5 K 2024-11-10T20:27:17,984 DEBUG [M:0;734d44df782b:45501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2011e2e7ce446759e8332e83fbcd74b as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2011e2e7ce446759e8332e83fbcd74b 2024-11-10T20:27:17,990 INFO [M:0;734d44df782b:45501 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2011e2e7ce446759e8332e83fbcd74b 2024-11-10T20:27:17,990 INFO [M:0;734d44df782b:45501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2011e2e7ce446759e8332e83fbcd74b, entries=6, sequenceid=59, filesize=6.1 K 2024-11-10T20:27:17,991 DEBUG [M:0;734d44df782b:45501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/59a591434100438ab4c914d99e273830 as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/59a591434100438ab4c914d99e273830 2024-11-10T20:27:17,998 INFO [M:0;734d44df782b:45501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/59a591434100438ab4c914d99e273830, entries=1, sequenceid=59, filesize=5.0 K 2024-11-10T20:27:17,999 DEBUG [M:0;734d44df782b:45501 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8b8369ae45184f92a4eec95f261e1489 as hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8b8369ae45184f92a4eec95f261e1489 2024-11-10T20:27:18,006 INFO [M:0;734d44df782b:45501 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8b8369ae45184f92a4eec95f261e1489, entries=1, sequenceid=59, filesize=4.9 K 2024-11-10T20:27:18,007 INFO [M:0;734d44df782b:45501 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 165ms, sequenceid=59, compaction requested=false 2024-11-10T20:27:18,009 INFO [M:0;734d44df782b:45501 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:18,009 DEBUG [M:0;734d44df782b:45501 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270437842Disabling compacts and flushes for region at 1731270437842Disabling writes for close at 1731270437842Obtaining lock to block concurrent updates at 1731270437843 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731270437843Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1731270437844 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731270437845 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731270437845Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731270437861 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731270437861Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731270437875 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731270437891 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731270437891Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731270437906 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731270437927 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731270437927Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731270437942 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731270437959 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731270437959Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b8e80bd: reopening flushed file at 1731270437974 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2612bfbc: reopening flushed file at 1731270437982 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bfc974f: reopening flushed file at 1731270437990 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53682029: reopening flushed file at 1731270437998 (+8 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 165ms, sequenceid=59, compaction requested=false at 1731270438007 (+9 ms)Writing region close event to WAL at 1731270438009 (+2 ms)Closed at 1731270438009 2024-11-10T20:27:18,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:18,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:18,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:18,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:18,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:18,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41181 is added to blk_1073741830_1006 (size=27985) 2024-11-10T20:27:18,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33157 is added to blk_1073741830_1006 (size=27985) 2024-11-10T20:27:18,014 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:27:18,014 INFO [M:0;734d44df782b:45501 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T20:27:18,014 INFO [M:0;734d44df782b:45501 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45501 2024-11-10T20:27:18,014 INFO [M:0;734d44df782b:45501 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:27:18,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:27:18,116 INFO [M:0;734d44df782b:45501 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:27:18,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45501-0x10193dc68fd0000, quorum=127.0.0.1:49357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:27:18,121 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:18,123 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:18,123 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:18,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:18,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:18,126 WARN [BP-1055372608-172.17.0.2-1731270339452 heartbeating to localhost/127.0.0.1:41563 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:27:18,126 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:27:18,126 WARN [BP-1055372608-172.17.0.2-1731270339452 heartbeating to localhost/127.0.0.1:41563 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055372608-172.17.0.2-1731270339452 (Datanode Uuid dbd83278-d45c-4301-91ee-84deff001f20) service to localhost/127.0.0.1:41563 2024-11-10T20:27:18,127 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:27:18,128 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/data/data3/current/BP-1055372608-172.17.0.2-1731270339452 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:18,128 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/data/data4/current/BP-1055372608-172.17.0.2-1731270339452 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:18,129 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:27:18,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:18,131 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:18,131 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:18,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:18,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:18,133 WARN [BP-1055372608-172.17.0.2-1731270339452 heartbeating to localhost/127.0.0.1:41563 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:27:18,133 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:27:18,133 WARN [BP-1055372608-172.17.0.2-1731270339452 heartbeating to localhost/127.0.0.1:41563 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055372608-172.17.0.2-1731270339452 (Datanode Uuid 88bd4023-f81d-41ab-b25e-705b0871e99f) service to localhost/127.0.0.1:41563 2024-11-10T20:27:18,133 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:27:18,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/data/data1/current/BP-1055372608-172.17.0.2-1731270339452 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:18,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/cluster_d657cfed-3ac3-11f3-c3d5-4944d34b8b08/data/data2/current/BP-1055372608-172.17.0.2-1731270339452 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:18,134 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:27:18,146 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:27:18,147 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:18,147 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:18,147 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:18,147 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:18,159 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T20:27:18,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T20:27:18,204 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=78 (was 12) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41563 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/734d44df782b:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41563 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: LeaseRenewer:jenkins@localhost:41563 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/734d44df782b:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41563 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41563 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/734d44df782b:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@4e6bf3cd java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41563 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:41563 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41563 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/734d44df782b:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=405 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=10 (was 0) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7863 (was 8432) 2024-11-10T20:27:18,208 INFO [regionserver/734d44df782b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:27:18,211 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=79, OpenFileDescriptor=405, MaxFileDescriptor=1048576, SystemLoadAverage=10, ProcessCount=11, AvailableMemoryMB=7862 2024-11-10T20:27:18,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T20:27:18,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.log.dir so I do NOT create it in target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b 2024-11-10T20:27:18,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/921aff8b-4af8-c4aa-f9f8-9a1cdaba12f0/hadoop.tmp.dir so I do NOT create it in target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b 2024-11-10T20:27:18,212 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969, deleteOnExit=true 2024-11-10T20:27:18,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T20:27:18,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/test.cache.data in system properties and HBase conf 2024-11-10T20:27:18,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T20:27:18,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.log.dir in system properties and HBase conf 2024-11-10T20:27:18,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T20:27:18,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T20:27:18,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T20:27:18,213 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T20:27:18,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/nfs.dump.dir in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/java.io.tmpdir in system properties and HBase conf 2024-11-10T20:27:18,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:27:18,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T20:27:18,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T20:27:18,230 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:27:18,304 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:18,311 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:18,312 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:18,312 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:18,312 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:27:18,313 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:18,313 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46d26a79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:18,313 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59505eb5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:18,431 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1edca743{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/java.io.tmpdir/jetty-localhost-43307-hadoop-hdfs-3_4_1-tests_jar-_-any-5422387342717785571/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:27:18,432 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c443180{HTTP/1.1, (http/1.1)}{localhost:43307} 2024-11-10T20:27:18,432 INFO [Time-limited test {}] server.Server(415): Started @100906ms 2024-11-10T20:27:18,446 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:27:18,513 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:18,516 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:18,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:18,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:18,517 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:27:18,518 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69a0f3c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:18,518 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@469dec96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:18,635 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a15ed6a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/java.io.tmpdir/jetty-localhost-41677-hadoop-hdfs-3_4_1-tests_jar-_-any-483976707472988471/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:18,635 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@b5aedfa{HTTP/1.1, (http/1.1)}{localhost:41677} 2024-11-10T20:27:18,635 INFO [Time-limited test {}] server.Server(415): Started @101109ms 2024-11-10T20:27:18,637 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:27:18,673 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:18,676 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:18,678 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:18,678 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:18,678 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:27:18,679 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@194f043a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:18,680 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@274298f3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:18,724 WARN [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/data/data1/current/BP-873426659-172.17.0.2-1731270438249/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:18,724 WARN [Thread-439 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/data/data2/current/BP-873426659-172.17.0.2-1731270438249/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:18,749 WARN [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:27:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9036ded4cdca0b97 with lease ID 0xfad1622013187895: Processing first storage report for DS-8641fc44-a45f-41d4-9524-562dea801b24 from datanode DatanodeRegistration(127.0.0.1:32893, datanodeUuid=e4a694c8-0596-4e91-8b8b-ca688765fd2e, infoPort=33809, infoSecurePort=0, ipcPort=46561, storageInfo=lv=-57;cid=testClusterID;nsid=1308272501;c=1731270438249) 2024-11-10T20:27:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9036ded4cdca0b97 with lease ID 0xfad1622013187895: from storage DS-8641fc44-a45f-41d4-9524-562dea801b24 node DatanodeRegistration(127.0.0.1:32893, datanodeUuid=e4a694c8-0596-4e91-8b8b-ca688765fd2e, infoPort=33809, infoSecurePort=0, ipcPort=46561, storageInfo=lv=-57;cid=testClusterID;nsid=1308272501;c=1731270438249), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9036ded4cdca0b97 with lease ID 0xfad1622013187895: Processing first storage report for DS-0215c325-3460-4dac-bdcd-ec79ea828a02 from datanode DatanodeRegistration(127.0.0.1:32893, datanodeUuid=e4a694c8-0596-4e91-8b8b-ca688765fd2e, infoPort=33809, infoSecurePort=0, ipcPort=46561, storageInfo=lv=-57;cid=testClusterID;nsid=1308272501;c=1731270438249) 2024-11-10T20:27:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9036ded4cdca0b97 with lease ID 0xfad1622013187895: from storage DS-0215c325-3460-4dac-bdcd-ec79ea828a02 node DatanodeRegistration(127.0.0.1:32893, datanodeUuid=e4a694c8-0596-4e91-8b8b-ca688765fd2e, infoPort=33809, infoSecurePort=0, ipcPort=46561, storageInfo=lv=-57;cid=testClusterID;nsid=1308272501;c=1731270438249), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:18,804 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@18492d7d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/java.io.tmpdir/jetty-localhost-36903-hadoop-hdfs-3_4_1-tests_jar-_-any-619260953353270668/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:18,804 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30ebe7e3{HTTP/1.1, (http/1.1)}{localhost:36903} 2024-11-10T20:27:18,804 INFO [Time-limited test {}] server.Server(415): Started @101278ms 2024-11-10T20:27:18,806 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:27:18,889 WARN [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/data/data3/current/BP-873426659-172.17.0.2-1731270438249/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:18,889 WARN [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/data/data4/current/BP-873426659-172.17.0.2-1731270438249/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:18,912 WARN [Thread-453 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:27:18,915 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x29316623594f65f4 with lease ID 0xfad1622013187896: Processing first storage report for DS-3c5398a4-751d-4ac1-8710-2945f0d143da from datanode DatanodeRegistration(127.0.0.1:33469, datanodeUuid=09aa3a14-72bd-4c9d-b249-43629cbce07a, infoPort=45227, infoSecurePort=0, ipcPort=37343, storageInfo=lv=-57;cid=testClusterID;nsid=1308272501;c=1731270438249) 2024-11-10T20:27:18,915 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x29316623594f65f4 with lease ID 0xfad1622013187896: from storage DS-3c5398a4-751d-4ac1-8710-2945f0d143da node DatanodeRegistration(127.0.0.1:33469, datanodeUuid=09aa3a14-72bd-4c9d-b249-43629cbce07a, infoPort=45227, infoSecurePort=0, ipcPort=37343, storageInfo=lv=-57;cid=testClusterID;nsid=1308272501;c=1731270438249), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T20:27:18,915 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x29316623594f65f4 with lease ID 0xfad1622013187896: Processing first storage report for DS-435f46a4-a1fb-45b4-a999-b67d03f914f6 from datanode DatanodeRegistration(127.0.0.1:33469, datanodeUuid=09aa3a14-72bd-4c9d-b249-43629cbce07a, infoPort=45227, infoSecurePort=0, ipcPort=37343, storageInfo=lv=-57;cid=testClusterID;nsid=1308272501;c=1731270438249) 2024-11-10T20:27:18,915 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x29316623594f65f4 with lease ID 0xfad1622013187896: from storage DS-435f46a4-a1fb-45b4-a999-b67d03f914f6 node DatanodeRegistration(127.0.0.1:33469, datanodeUuid=09aa3a14-72bd-4c9d-b249-43629cbce07a, infoPort=45227, infoSecurePort=0, ipcPort=37343, storageInfo=lv=-57;cid=testClusterID;nsid=1308272501;c=1731270438249), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:18,935 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b 2024-11-10T20:27:18,937 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/zookeeper_0, clientPort=65016, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T20:27:18,938 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65016 2024-11-10T20:27:18,938 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:18,940 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:18,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:27:18,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:27:18,952 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac with version=8 2024-11-10T20:27:18,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/hbase-staging 2024-11-10T20:27:18,954 INFO [Time-limited test {}] client.ConnectionUtils(128): master/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:27:18,954 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:18,955 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:18,955 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:27:18,955 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:18,955 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:27:18,955 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T20:27:18,955 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:27:18,956 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43579 2024-11-10T20:27:18,957 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43579 connecting to ZooKeeper ensemble=127.0.0.1:65016 2024-11-10T20:27:18,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:435790x0, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:27:18,963 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43579-0x10193dde5810000 connected 2024-11-10T20:27:18,985 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:18,987 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:18,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:18,991 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac, hbase.cluster.distributed=false 2024-11-10T20:27:18,993 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:27:18,993 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43579 2024-11-10T20:27:18,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43579 2024-11-10T20:27:18,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43579 2024-11-10T20:27:18,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43579 2024-11-10T20:27:18,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43579 2024-11-10T20:27:19,015 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:27:19,015 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:19,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:19,016 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:27:19,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:19,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:27:19,016 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T20:27:19,016 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:27:19,017 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39957 2024-11-10T20:27:19,018 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39957 connecting to ZooKeeper ensemble=127.0.0.1:65016 2024-11-10T20:27:19,019 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:19,021 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:19,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399570x0, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:27:19,027 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39957-0x10193dde5810001 connected 2024-11-10T20:27:19,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:19,027 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T20:27:19,030 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T20:27:19,031 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T20:27:19,033 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:27:19,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39957 2024-11-10T20:27:19,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39957 2024-11-10T20:27:19,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39957 2024-11-10T20:27:19,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39957 2024-11-10T20:27:19,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39957 2024-11-10T20:27:19,051 DEBUG [M:0;734d44df782b:43579 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;734d44df782b:43579 2024-11-10T20:27:19,051 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/734d44df782b,43579,1731270438954 2024-11-10T20:27:19,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:27:19,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:27:19,054 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/734d44df782b,43579,1731270438954 2024-11-10T20:27:19,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T20:27:19,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,056 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T20:27:19,056 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/734d44df782b,43579,1731270438954 from backup master directory 2024-11-10T20:27:19,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/734d44df782b,43579,1731270438954 2024-11-10T20:27:19,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:27:19,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:27:19,058 WARN [master/734d44df782b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:27:19,058 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=734d44df782b,43579,1731270438954 2024-11-10T20:27:19,066 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/hbase.id] with ID: 05459968-4f59-4d37-8c59-8a9c2f9547da 2024-11-10T20:27:19,066 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/.tmp/hbase.id 2024-11-10T20:27:19,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:27:19,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:27:19,074 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/.tmp/hbase.id]:[hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/hbase.id] 2024-11-10T20:27:19,090 INFO [master/734d44df782b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:19,090 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T20:27:19,092 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-10T20:27:19,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:27:19,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:27:19,107 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:27:19,109 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T20:27:19,109 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:27:19,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:27:19,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:27:19,123 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store 2024-11-10T20:27:19,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:27:19,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:27:19,133 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:19,133 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:27:19,133 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:19,133 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:19,133 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:27:19,133 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:19,133 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:19,133 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270439133Disabling compacts and flushes for region at 1731270439133Disabling writes for close at 1731270439133Writing region close event to WAL at 1731270439133Closed at 1731270439133 2024-11-10T20:27:19,134 WARN [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/.initializing 2024-11-10T20:27:19,134 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/WALs/734d44df782b,43579,1731270438954 2024-11-10T20:27:19,138 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C43579%2C1731270438954, suffix=, logDir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/WALs/734d44df782b,43579,1731270438954, archiveDir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/oldWALs, maxLogs=10 2024-11-10T20:27:19,138 INFO [master/734d44df782b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C43579%2C1731270438954.1731270439138 2024-11-10T20:27:19,144 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/WALs/734d44df782b,43579,1731270438954/734d44df782b%2C43579%2C1731270438954.1731270439138 2024-11-10T20:27:19,145 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33809:33809),(127.0.0.1/127.0.0.1:45227:45227)] 2024-11-10T20:27:19,148 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:27:19,148 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:19,148 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,148 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T20:27:19,154 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:19,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,156 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T20:27:19,156 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,156 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:27:19,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T20:27:19,159 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:27:19,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T20:27:19,161 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:27:19,162 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,163 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,163 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,165 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,165 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,165 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T20:27:19,167 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:19,170 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:27:19,170 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=781875, jitterRate=-0.005795389413833618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T20:27:19,172 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731270439148Initializing all the Stores at 1731270439149 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270439149Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270439152 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270439152Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270439152Cleaning up temporary data from old regions at 1731270439165 (+13 ms)Region opened successfully at 1731270439171 (+6 ms) 2024-11-10T20:27:19,172 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T20:27:19,176 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f39108a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:27:19,177 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T20:27:19,177 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T20:27:19,177 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T20:27:19,177 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T20:27:19,178 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T20:27:19,178 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T20:27:19,178 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T20:27:19,181 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T20:27:19,181 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T20:27:19,183 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T20:27:19,183 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T20:27:19,184 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T20:27:19,185 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T20:27:19,185 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T20:27:19,186 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T20:27:19,187 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T20:27:19,188 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T20:27:19,189 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T20:27:19,191 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T20:27:19,192 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T20:27:19,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:19,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:19,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,194 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=734d44df782b,43579,1731270438954, sessionid=0x10193dde5810000, setting cluster-up flag (Was=false) 2024-11-10T20:27:19,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,202 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T20:27:19,203 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,43579,1731270438954 2024-11-10T20:27:19,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,210 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T20:27:19,211 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,43579,1731270438954 2024-11-10T20:27:19,213 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T20:27:19,215 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T20:27:19,215 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T20:27:19,215 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T20:27:19,215 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 734d44df782b,43579,1731270438954 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T20:27:19,217 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:27:19,217 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:27:19,217 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:27:19,217 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:27:19,217 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/734d44df782b:0, corePoolSize=10, maxPoolSize=10 2024-11-10T20:27:19,217 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,217 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:27:19,217 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,219 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:27:19,219 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T20:27:19,220 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731270469220 2024-11-10T20:27:19,220 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T20:27:19,220 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T20:27:19,221 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T20:27:19,221 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T20:27:19,221 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T20:27:19,221 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T20:27:19,221 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,221 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,221 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T20:27:19,221 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T20:27:19,221 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T20:27:19,221 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T20:27:19,222 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T20:27:19,222 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T20:27:19,222 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270439222,5,FailOnTimeoutGroup] 2024-11-10T20:27:19,223 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270439222,5,FailOnTimeoutGroup] 2024-11-10T20:27:19,223 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,223 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T20:27:19,223 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,223 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:27:19,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:27:19,230 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T20:27:19,231 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac 2024-11-10T20:27:19,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:27:19,240 INFO [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(746): ClusterId : 05459968-4f59-4d37-8c59-8a9c2f9547da 2024-11-10T20:27:19,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:27:19,240 DEBUG [RS:0;734d44df782b:39957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T20:27:19,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:19,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:27:19,242 DEBUG [RS:0;734d44df782b:39957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T20:27:19,242 DEBUG [RS:0;734d44df782b:39957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T20:27:19,243 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:27:19,243 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:19,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:27:19,245 DEBUG [RS:0;734d44df782b:39957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T20:27:19,245 DEBUG [RS:0;734d44df782b:39957 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ea5091, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:27:19,246 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:27:19,246 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:19,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:27:19,249 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:27:19,249 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:19,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:27:19,251 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:27:19,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:19,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:27:19,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740 2024-11-10T20:27:19,253 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740 2024-11-10T20:27:19,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:27:19,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:27:19,255 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:27:19,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:27:19,259 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:27:19,259 DEBUG [RS:0;734d44df782b:39957 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;734d44df782b:39957 2024-11-10T20:27:19,260 INFO [RS:0;734d44df782b:39957 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T20:27:19,260 INFO [RS:0;734d44df782b:39957 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T20:27:19,260 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=770006, jitterRate=-0.020887374877929688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:27:19,260 DEBUG [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T20:27:19,261 INFO [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(2659): reportForDuty to master=734d44df782b,43579,1731270438954 with port=39957, startcode=1731270439015 2024-11-10T20:27:19,261 DEBUG [RS:0;734d44df782b:39957 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T20:27:19,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731270439240Initializing all the Stores at 1731270439241 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270439241Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270439241Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270439241Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270439241Cleaning up temporary data from old regions at 1731270439254 (+13 ms)Region opened successfully at 1731270439261 (+7 ms) 2024-11-10T20:27:19,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:27:19,261 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:27:19,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:27:19,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:27:19,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:27:19,262 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:27:19,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270439261Disabling compacts and flushes for region at 1731270439261Disabling writes for close at 1731270439261Writing region close event to WAL at 1731270439262 (+1 ms)Closed at 1731270439262 2024-11-10T20:27:19,264 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:27:19,264 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T20:27:19,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T20:27:19,264 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43555, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T20:27:19,265 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 734d44df782b,39957,1731270439015 2024-11-10T20:27:19,265 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43579 {}] master.ServerManager(517): Registering regionserver=734d44df782b,39957,1731270439015 2024-11-10T20:27:19,266 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:27:19,267 DEBUG [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac 2024-11-10T20:27:19,267 DEBUG [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38377 2024-11-10T20:27:19,267 DEBUG [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T20:27:19,268 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T20:27:19,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:27:19,270 DEBUG [RS:0;734d44df782b:39957 {}] zookeeper.ZKUtil(111): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/734d44df782b,39957,1731270439015 2024-11-10T20:27:19,270 WARN [RS:0;734d44df782b:39957 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:27:19,270 INFO [RS:0;734d44df782b:39957 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:27:19,270 DEBUG [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/WALs/734d44df782b,39957,1731270439015 2024-11-10T20:27:19,270 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [734d44df782b,39957,1731270439015] 2024-11-10T20:27:19,274 INFO [RS:0;734d44df782b:39957 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T20:27:19,277 INFO [RS:0;734d44df782b:39957 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T20:27:19,280 INFO [RS:0;734d44df782b:39957 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T20:27:19,280 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,281 INFO [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T20:27:19,282 INFO [RS:0;734d44df782b:39957 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T20:27:19,282 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,282 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,282 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,282 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,282 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,282 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,282 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:27:19,282 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,283 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,283 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,283 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,283 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,283 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:19,283 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:27:19,283 DEBUG [RS:0;734d44df782b:39957 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:27:19,284 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,284 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,284 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,284 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,284 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,284 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,39957,1731270439015-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:27:19,301 INFO [RS:0;734d44df782b:39957 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T20:27:19,301 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,39957,1731270439015-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,301 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,301 INFO [RS:0;734d44df782b:39957 {}] regionserver.Replication(171): 734d44df782b,39957,1731270439015 started 2024-11-10T20:27:19,317 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,317 INFO [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(1482): Serving as 734d44df782b,39957,1731270439015, RpcServer on 734d44df782b/172.17.0.2:39957, sessionid=0x10193dde5810001 2024-11-10T20:27:19,317 DEBUG [RS:0;734d44df782b:39957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T20:27:19,317 DEBUG [RS:0;734d44df782b:39957 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 734d44df782b,39957,1731270439015 2024-11-10T20:27:19,317 DEBUG [RS:0;734d44df782b:39957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,39957,1731270439015' 2024-11-10T20:27:19,317 DEBUG [RS:0;734d44df782b:39957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T20:27:19,318 DEBUG [RS:0;734d44df782b:39957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T20:27:19,319 DEBUG [RS:0;734d44df782b:39957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T20:27:19,319 DEBUG [RS:0;734d44df782b:39957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T20:27:19,319 DEBUG [RS:0;734d44df782b:39957 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 734d44df782b,39957,1731270439015 2024-11-10T20:27:19,319 DEBUG [RS:0;734d44df782b:39957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,39957,1731270439015' 2024-11-10T20:27:19,319 DEBUG [RS:0;734d44df782b:39957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T20:27:19,319 DEBUG [RS:0;734d44df782b:39957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T20:27:19,320 DEBUG [RS:0;734d44df782b:39957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T20:27:19,320 INFO [RS:0;734d44df782b:39957 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T20:27:19,320 INFO [RS:0;734d44df782b:39957 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T20:27:19,418 WARN [734d44df782b:43579 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T20:27:19,422 INFO [RS:0;734d44df782b:39957 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C39957%2C1731270439015, suffix=, logDir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/WALs/734d44df782b,39957,1731270439015, archiveDir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/oldWALs, maxLogs=32 2024-11-10T20:27:19,425 INFO [RS:0;734d44df782b:39957 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C39957%2C1731270439015.1731270439424 2024-11-10T20:27:19,431 INFO [RS:0;734d44df782b:39957 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/WALs/734d44df782b,39957,1731270439015/734d44df782b%2C39957%2C1731270439015.1731270439424 2024-11-10T20:27:19,432 DEBUG [RS:0;734d44df782b:39957 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45227:45227),(127.0.0.1/127.0.0.1:33809:33809)] 2024-11-10T20:27:19,669 DEBUG [734d44df782b:43579 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T20:27:19,669 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=734d44df782b,39957,1731270439015 2024-11-10T20:27:19,671 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,39957,1731270439015, state=OPENING 2024-11-10T20:27:19,672 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T20:27:19,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,675 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:27:19,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:27:19,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,39957,1731270439015}] 2024-11-10T20:27:19,675 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:27:19,828 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T20:27:19,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50865, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T20:27:19,836 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T20:27:19,836 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:27:19,838 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C39957%2C1731270439015.meta, suffix=.meta, logDir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/WALs/734d44df782b,39957,1731270439015, archiveDir=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/oldWALs, maxLogs=32 2024-11-10T20:27:19,840 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C39957%2C1731270439015.meta.1731270439840.meta 2024-11-10T20:27:19,847 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/WALs/734d44df782b,39957,1731270439015/734d44df782b%2C39957%2C1731270439015.meta.1731270439840.meta 2024-11-10T20:27:19,847 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33809:33809),(127.0.0.1/127.0.0.1:45227:45227)] 2024-11-10T20:27:19,849 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:27:19,849 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T20:27:19,849 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T20:27:19,849 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T20:27:19,849 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T20:27:19,849 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:19,850 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T20:27:19,850 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T20:27:19,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:27:19,853 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:27:19,853 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,854 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:19,854 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:27:19,855 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:27:19,855 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:19,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:27:19,857 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:27:19,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:19,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:27:19,859 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:27:19,859 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:19,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:19,860 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:27:19,861 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740 2024-11-10T20:27:19,862 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740 2024-11-10T20:27:19,864 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:27:19,864 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:27:19,865 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:27:19,866 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:27:19,867 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=845410, jitterRate=0.07499444484710693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:27:19,867 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T20:27:19,869 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731270439850Writing region info on filesystem at 1731270439850Initializing all the Stores at 1731270439851 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270439851Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270439852 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270439852Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270439852Cleaning up temporary data from old regions at 1731270439864 (+12 ms)Running coprocessor post-open hooks at 1731270439867 (+3 ms)Region opened successfully at 1731270439869 (+2 ms) 2024-11-10T20:27:19,870 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731270439828 2024-11-10T20:27:19,873 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T20:27:19,873 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T20:27:19,874 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,39957,1731270439015 2024-11-10T20:27:19,875 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,39957,1731270439015, state=OPEN 2024-11-10T20:27:19,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:27:19,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:27:19,880 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=734d44df782b,39957,1731270439015 2024-11-10T20:27:19,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:27:19,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:27:19,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T20:27:19,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,39957,1731270439015 in 205 msec 2024-11-10T20:27:19,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T20:27:19,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 619 msec 2024-11-10T20:27:19,887 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:27:19,887 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T20:27:19,889 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:27:19,889 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,39957,1731270439015, seqNum=-1] 2024-11-10T20:27:19,890 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:27:19,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57871, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:27:19,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 683 msec 2024-11-10T20:27:19,898 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731270439898, completionTime=-1 2024-11-10T20:27:19,898 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T20:27:19,899 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T20:27:19,901 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T20:27:19,901 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731270499901 2024-11-10T20:27:19,901 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731270559901 2024-11-10T20:27:19,901 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-10T20:27:19,901 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43579,1731270438954-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,901 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43579,1731270438954-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,901 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43579,1731270438954-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,901 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-734d44df782b:43579, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,901 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,902 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,903 DEBUG [master/734d44df782b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T20:27:19,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.849sec 2024-11-10T20:27:19,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T20:27:19,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T20:27:19,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T20:27:19,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T20:27:19,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T20:27:19,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43579,1731270438954-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:27:19,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43579,1731270438954-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T20:27:19,910 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T20:27:19,910 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T20:27:19,910 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43579,1731270438954-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:19,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@210f4ee9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:27:19,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 734d44df782b,43579,-1 for getting cluster id 2024-11-10T20:27:19,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T20:27:19,942 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '05459968-4f59-4d37-8c59-8a9c2f9547da' 2024-11-10T20:27:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T20:27:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "05459968-4f59-4d37-8c59-8a9c2f9547da" 2024-11-10T20:27:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f7577d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:27:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [734d44df782b,43579,-1] 2024-11-10T20:27:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T20:27:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:19,946 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T20:27:19,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf88cd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:27:19,948 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:27:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,39957,1731270439015, seqNum=-1] 2024-11-10T20:27:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:27:19,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:27:19,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=734d44df782b,43579,1731270438954 2024-11-10T20:27:19,954 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:19,957 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T20:27:19,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T20:27:19,957 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:27:19,957 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:19,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:19,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:19,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T20:27:19,958 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T20:27:19,958 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=571199657, stopped=false 2024-11-10T20:27:19,958 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=734d44df782b,43579,1731270438954 2024-11-10T20:27:19,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:19,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:19,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:19,960 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:27:19,960 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:27:19,960 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:19,961 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:19,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:19,961 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:19,961 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '734d44df782b,39957,1731270439015' ***** 2024-11-10T20:27:19,961 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T20:27:19,961 INFO [RS:0;734d44df782b:39957 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T20:27:19,961 INFO [RS:0;734d44df782b:39957 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T20:27:19,961 INFO [RS:0;734d44df782b:39957 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T20:27:19,962 INFO [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(959): stopping server 734d44df782b,39957,1731270439015 2024-11-10T20:27:19,962 INFO [RS:0;734d44df782b:39957 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:27:19,962 INFO [RS:0;734d44df782b:39957 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;734d44df782b:39957. 2024-11-10T20:27:19,962 DEBUG [RS:0;734d44df782b:39957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:19,962 DEBUG [RS:0;734d44df782b:39957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:19,962 INFO [RS:0;734d44df782b:39957 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T20:27:19,962 INFO [RS:0;734d44df782b:39957 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T20:27:19,962 INFO [RS:0;734d44df782b:39957 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T20:27:19,962 INFO [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T20:27:19,962 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T20:27:19,962 INFO [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T20:27:19,963 DEBUG [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T20:27:19,963 DEBUG [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T20:27:19,963 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:27:19,963 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:27:19,963 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:27:19,963 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:27:19,963 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:27:19,963 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-10T20:27:19,989 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740/.tmp/ns/203cdf1f4384426dad82cf476874a2e4 is 43, key is default/ns:d/1731270439892/Put/seqid=0 2024-11-10T20:27:19,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741835_1011 (size=5153) 2024-11-10T20:27:19,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741835_1011 (size=5153) 2024-11-10T20:27:19,996 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740/.tmp/ns/203cdf1f4384426dad82cf476874a2e4 2024-11-10T20:27:20,004 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740/.tmp/ns/203cdf1f4384426dad82cf476874a2e4 as hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740/ns/203cdf1f4384426dad82cf476874a2e4 2024-11-10T20:27:20,011 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740/ns/203cdf1f4384426dad82cf476874a2e4, entries=2, sequenceid=6, filesize=5.0 K 2024-11-10T20:27:20,013 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=6, compaction requested=false 2024-11-10T20:27:20,018 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T20:27:20,018 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:27:20,019 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:27:20,019 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270439963Running coprocessor pre-close hooks at 1731270439963Disabling compacts and flushes for region at 1731270439963Disabling writes for close at 1731270439963Obtaining lock to block concurrent updates at 1731270439963Preparing flush snapshotting stores in 1588230740 at 1731270439963Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731270439963Flushing stores of hbase:meta,,1.1588230740 at 1731270439964 (+1 ms)Flushing 1588230740/ns: creating writer at 1731270439965 (+1 ms)Flushing 1588230740/ns: appending metadata at 1731270439988 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1731270439988Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ba2bf8c: reopening flushed file at 1731270440003 (+15 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=6, compaction requested=false at 1731270440013 (+10 ms)Writing region close event to WAL at 1731270440014 (+1 ms)Running coprocessor post-close hooks at 1731270440018 (+4 ms)Closed at 1731270440019 (+1 ms) 2024-11-10T20:27:20,019 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T20:27:20,163 INFO [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(976): stopping server 734d44df782b,39957,1731270439015; all regions closed. 2024-11-10T20:27:20,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741834_1010 (size=1152) 2024-11-10T20:27:20,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741834_1010 (size=1152) 2024-11-10T20:27:20,170 DEBUG [RS:0;734d44df782b:39957 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/oldWALs 2024-11-10T20:27:20,170 INFO [RS:0;734d44df782b:39957 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C39957%2C1731270439015.meta:.meta(num 1731270439840) 2024-11-10T20:27:20,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741833_1009 (size=93) 2024-11-10T20:27:20,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741833_1009 (size=93) 2024-11-10T20:27:20,176 DEBUG [RS:0;734d44df782b:39957 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/oldWALs 2024-11-10T20:27:20,176 INFO [RS:0;734d44df782b:39957 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C39957%2C1731270439015:(num 1731270439424) 2024-11-10T20:27:20,176 DEBUG [RS:0;734d44df782b:39957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:20,176 INFO [RS:0;734d44df782b:39957 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:27:20,177 INFO [RS:0;734d44df782b:39957 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:27:20,177 INFO [RS:0;734d44df782b:39957 {}] hbase.ChoreService(370): Chore service for: regionserver/734d44df782b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T20:27:20,177 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:27:20,177 INFO [RS:0;734d44df782b:39957 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:27:20,177 INFO [RS:0;734d44df782b:39957 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39957 2024-11-10T20:27:20,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:27:20,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/734d44df782b,39957,1731270439015 2024-11-10T20:27:20,180 INFO [RS:0;734d44df782b:39957 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:27:20,182 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [734d44df782b,39957,1731270439015] 2024-11-10T20:27:20,183 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/734d44df782b,39957,1731270439015 already deleted, retry=false 2024-11-10T20:27:20,183 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 734d44df782b,39957,1731270439015 expired; onlineServers=0 2024-11-10T20:27:20,183 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '734d44df782b,43579,1731270438954' ***** 2024-11-10T20:27:20,183 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T20:27:20,183 INFO [M:0;734d44df782b:43579 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:27:20,183 INFO [M:0;734d44df782b:43579 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:27:20,183 DEBUG [M:0;734d44df782b:43579 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T20:27:20,183 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T20:27:20,184 DEBUG [M:0;734d44df782b:43579 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T20:27:20,184 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270439222 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270439222,5,FailOnTimeoutGroup] 2024-11-10T20:27:20,184 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270439222 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270439222,5,FailOnTimeoutGroup] 2024-11-10T20:27:20,184 INFO [M:0;734d44df782b:43579 {}] hbase.ChoreService(370): Chore service for: master/734d44df782b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T20:27:20,184 INFO [M:0;734d44df782b:43579 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:27:20,184 DEBUG [M:0;734d44df782b:43579 {}] master.HMaster(1795): Stopping service threads 2024-11-10T20:27:20,184 INFO [M:0;734d44df782b:43579 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T20:27:20,184 INFO [M:0;734d44df782b:43579 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:27:20,184 INFO [M:0;734d44df782b:43579 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T20:27:20,184 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T20:27:20,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T20:27:20,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:20,185 DEBUG [M:0;734d44df782b:43579 {}] zookeeper.ZKUtil(347): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T20:27:20,185 WARN [M:0;734d44df782b:43579 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T20:27:20,186 INFO [M:0;734d44df782b:43579 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/.lastflushedseqids 2024-11-10T20:27:20,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741836_1012 (size=99) 2024-11-10T20:27:20,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741836_1012 (size=99) 2024-11-10T20:27:20,193 INFO [M:0;734d44df782b:43579 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T20:27:20,193 INFO [M:0;734d44df782b:43579 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T20:27:20,193 DEBUG [M:0;734d44df782b:43579 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:27:20,194 INFO [M:0;734d44df782b:43579 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:20,194 DEBUG [M:0;734d44df782b:43579 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:20,194 DEBUG [M:0;734d44df782b:43579 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:27:20,194 DEBUG [M:0;734d44df782b:43579 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:20,195 INFO [M:0;734d44df782b:43579 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-10T20:27:20,213 DEBUG [M:0;734d44df782b:43579 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9f9d95e40daa40d5bdc2113ec624c16a is 82, key is hbase:meta,,1/info:regioninfo/1731270439874/Put/seqid=0 2024-11-10T20:27:20,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741837_1013 (size=5672) 2024-11-10T20:27:20,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741837_1013 (size=5672) 2024-11-10T20:27:20,220 INFO [M:0;734d44df782b:43579 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9f9d95e40daa40d5bdc2113ec624c16a 2024-11-10T20:27:20,243 DEBUG [M:0;734d44df782b:43579 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80812a60b9714789a1f8488312b5dd72 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731270439897/Put/seqid=0 2024-11-10T20:27:20,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741838_1014 (size=5275) 2024-11-10T20:27:20,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741838_1014 (size=5275) 2024-11-10T20:27:20,249 INFO [M:0;734d44df782b:43579 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80812a60b9714789a1f8488312b5dd72 2024-11-10T20:27:20,271 DEBUG [M:0;734d44df782b:43579 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3208f8e9563146ed9f8d4cc3222e1ee7 is 69, key is 734d44df782b,39957,1731270439015/rs:state/1731270439265/Put/seqid=0 2024-11-10T20:27:20,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741839_1015 (size=5156) 2024-11-10T20:27:20,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741839_1015 (size=5156) 2024-11-10T20:27:20,277 INFO [M:0;734d44df782b:43579 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3208f8e9563146ed9f8d4cc3222e1ee7 2024-11-10T20:27:20,282 INFO [RS:0;734d44df782b:39957 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:27:20,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:27:20,282 INFO [RS:0;734d44df782b:39957 {}] regionserver.HRegionServer(1031): Exiting; stopping=734d44df782b,39957,1731270439015; zookeeper connection closed. 2024-11-10T20:27:20,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39957-0x10193dde5810001, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:27:20,282 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5b7d0464 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5b7d0464 2024-11-10T20:27:20,282 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T20:27:20,298 DEBUG [M:0;734d44df782b:43579 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f59eb38dff454aebb3ced203922b4123 is 52, key is load_balancer_on/state:d/1731270439955/Put/seqid=0 2024-11-10T20:27:20,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741840_1016 (size=5056) 2024-11-10T20:27:20,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741840_1016 (size=5056) 2024-11-10T20:27:20,304 INFO [M:0;734d44df782b:43579 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f59eb38dff454aebb3ced203922b4123 2024-11-10T20:27:20,312 DEBUG [M:0;734d44df782b:43579 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9f9d95e40daa40d5bdc2113ec624c16a as hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9f9d95e40daa40d5bdc2113ec624c16a 2024-11-10T20:27:20,318 INFO [M:0;734d44df782b:43579 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9f9d95e40daa40d5bdc2113ec624c16a, entries=8, sequenceid=29, filesize=5.5 K 2024-11-10T20:27:20,319 DEBUG [M:0;734d44df782b:43579 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80812a60b9714789a1f8488312b5dd72 as hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/80812a60b9714789a1f8488312b5dd72 2024-11-10T20:27:20,325 INFO [M:0;734d44df782b:43579 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/80812a60b9714789a1f8488312b5dd72, entries=3, sequenceid=29, filesize=5.2 K 2024-11-10T20:27:20,326 DEBUG [M:0;734d44df782b:43579 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3208f8e9563146ed9f8d4cc3222e1ee7 as hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3208f8e9563146ed9f8d4cc3222e1ee7 2024-11-10T20:27:20,333 INFO [M:0;734d44df782b:43579 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3208f8e9563146ed9f8d4cc3222e1ee7, entries=1, sequenceid=29, filesize=5.0 K 2024-11-10T20:27:20,334 DEBUG [M:0;734d44df782b:43579 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f59eb38dff454aebb3ced203922b4123 as hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f59eb38dff454aebb3ced203922b4123 2024-11-10T20:27:20,340 INFO [M:0;734d44df782b:43579 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38377/user/jenkins/test-data/64e40d84-a238-4432-5823-43bd3e3011ac/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f59eb38dff454aebb3ced203922b4123, entries=1, sequenceid=29, filesize=4.9 K 2024-11-10T20:27:20,342 INFO [M:0;734d44df782b:43579 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 148ms, sequenceid=29, compaction requested=false 2024-11-10T20:27:20,343 INFO [M:0;734d44df782b:43579 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:20,344 DEBUG [M:0;734d44df782b:43579 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270440193Disabling compacts and flushes for region at 1731270440193Disabling writes for close at 1731270440194 (+1 ms)Obtaining lock to block concurrent updates at 1731270440195 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731270440195Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731270440195Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731270440196 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731270440196Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731270440213 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731270440213Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731270440226 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731270440243 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731270440243Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731270440255 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731270440270 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731270440270Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731270440282 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731270440298 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731270440298Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b49c8c9: reopening flushed file at 1731270440310 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d69f206: reopening flushed file at 1731270440318 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5120f889: reopening flushed file at 1731270440325 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fc23c94: reopening flushed file at 1731270440333 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 148ms, sequenceid=29, compaction requested=false at 1731270440342 (+9 ms)Writing region close event to WAL at 1731270440343 (+1 ms)Closed at 1731270440343 2024-11-10T20:27:20,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:20,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33469 is added to blk_1073741830_1006 (size=10311) 2024-11-10T20:27:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32893 is added to blk_1073741830_1006 (size=10311) 2024-11-10T20:27:20,349 INFO [M:0;734d44df782b:43579 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T20:27:20,349 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:27:20,349 INFO [M:0;734d44df782b:43579 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43579 2024-11-10T20:27:20,349 INFO [M:0;734d44df782b:43579 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:27:20,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:27:20,451 INFO [M:0;734d44df782b:43579 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:27:20,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43579-0x10193dde5810000, quorum=127.0.0.1:65016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:27:20,454 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@18492d7d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:20,454 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30ebe7e3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:20,454 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:20,454 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@274298f3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:20,455 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@194f043a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:20,456 WARN [BP-873426659-172.17.0.2-1731270438249 heartbeating to localhost/127.0.0.1:38377 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:27:20,456 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:27:20,456 WARN [BP-873426659-172.17.0.2-1731270438249 heartbeating to localhost/127.0.0.1:38377 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-873426659-172.17.0.2-1731270438249 (Datanode Uuid 09aa3a14-72bd-4c9d-b249-43629cbce07a) service to localhost/127.0.0.1:38377 2024-11-10T20:27:20,456 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:27:20,457 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/data/data3/current/BP-873426659-172.17.0.2-1731270438249 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:20,457 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/data/data4/current/BP-873426659-172.17.0.2-1731270438249 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:20,457 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:27:20,459 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a15ed6a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:20,459 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@b5aedfa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:20,460 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:20,460 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@469dec96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:20,460 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69a0f3c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:20,461 WARN [BP-873426659-172.17.0.2-1731270438249 heartbeating to localhost/127.0.0.1:38377 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:27:20,461 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:27:20,461 WARN [BP-873426659-172.17.0.2-1731270438249 heartbeating to localhost/127.0.0.1:38377 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-873426659-172.17.0.2-1731270438249 (Datanode Uuid e4a694c8-0596-4e91-8b8b-ca688765fd2e) service to localhost/127.0.0.1:38377 2024-11-10T20:27:20,461 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:27:20,462 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/data/data1/current/BP-873426659-172.17.0.2-1731270438249 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:20,462 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/cluster_cd90ba10-b33a-5ee5-e36c-dcfb8559b969/data/data2/current/BP-873426659-172.17.0.2-1731270438249 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:20,462 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:27:20,468 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1edca743{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:27:20,469 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c443180{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:20,469 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:20,469 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59505eb5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:20,469 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46d26a79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:20,475 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T20:27:20,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T20:27:20,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T20:27:20,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.log.dir so I do NOT create it in target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394 2024-11-10T20:27:20,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8063dbb-f809-79ec-f4b1-013f77bdeb6b/hadoop.tmp.dir so I do NOT create it in target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394 2024-11-10T20:27:20,492 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8, deleteOnExit=true 2024-11-10T20:27:20,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T20:27:20,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/test.cache.data in system properties and HBase conf 2024-11-10T20:27:20,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T20:27:20,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir in system properties and HBase conf 2024-11-10T20:27:20,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T20:27:20,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T20:27:20,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T20:27:20,492 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/nfs.dump.dir in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/java.io.tmpdir in system properties and HBase conf 2024-11-10T20:27:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:27:20,494 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T20:27:20,494 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T20:27:20,508 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:27:20,577 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:20,585 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:20,588 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:20,588 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:20,588 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:27:20,589 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:20,590 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@142d24a0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:20,590 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42b52d44{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:20,707 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c00ef51{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/java.io.tmpdir/jetty-localhost-33565-hadoop-hdfs-3_4_1-tests_jar-_-any-18196280972078997848/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:27:20,708 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@73b9709e{HTTP/1.1, (http/1.1)}{localhost:33565} 2024-11-10T20:27:20,708 INFO [Time-limited test {}] server.Server(415): Started @103182ms 2024-11-10T20:27:20,722 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:27:20,786 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:20,790 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:20,791 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:20,791 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:20,791 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:27:20,791 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@65dec1b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:20,792 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a107105{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:20,910 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6de86657{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/java.io.tmpdir/jetty-localhost-33357-hadoop-hdfs-3_4_1-tests_jar-_-any-3577198210757612774/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:20,910 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6787773a{HTTP/1.1, (http/1.1)}{localhost:33357} 2024-11-10T20:27:20,910 INFO [Time-limited test {}] server.Server(415): Started @103385ms 2024-11-10T20:27:20,912 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:27:20,944 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:20,948 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:20,950 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:20,950 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:20,950 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:27:20,951 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@66182b08{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:20,952 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7eee535{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:20,998 WARN [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data1/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:20,998 WARN [Thread-658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data2/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:21,017 WARN [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:27:21,020 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb75b4a2c233b838d with lease ID 0xa6dd06c378e0d824: Processing first storage report for DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b from datanode DatanodeRegistration(127.0.0.1:42605, datanodeUuid=8ba45455-c138-4c8c-bb3f-c05acbb2cf33, infoPort=40605, infoSecurePort=0, ipcPort=38207, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:21,020 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb75b4a2c233b838d with lease ID 0xa6dd06c378e0d824: from storage DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b node DatanodeRegistration(127.0.0.1:42605, datanodeUuid=8ba45455-c138-4c8c-bb3f-c05acbb2cf33, infoPort=40605, infoSecurePort=0, ipcPort=38207, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:21,020 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb75b4a2c233b838d with lease ID 0xa6dd06c378e0d824: Processing first storage report for DS-00986830-8686-4cbb-8852-2a562c54e658 from datanode DatanodeRegistration(127.0.0.1:42605, datanodeUuid=8ba45455-c138-4c8c-bb3f-c05acbb2cf33, infoPort=40605, infoSecurePort=0, ipcPort=38207, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:21,020 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb75b4a2c233b838d with lease ID 0xa6dd06c378e0d824: from storage DS-00986830-8686-4cbb-8852-2a562c54e658 node DatanodeRegistration(127.0.0.1:42605, datanodeUuid=8ba45455-c138-4c8c-bb3f-c05acbb2cf33, infoPort=40605, infoSecurePort=0, ipcPort=38207, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:21,070 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f2859b3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/java.io.tmpdir/jetty-localhost-45755-hadoop-hdfs-3_4_1-tests_jar-_-any-17381649565762210248/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:21,071 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3b918d2a{HTTP/1.1, (http/1.1)}{localhost:45755} 2024-11-10T20:27:21,071 INFO [Time-limited test {}] server.Server(415): Started @103545ms 2024-11-10T20:27:21,073 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:27:21,160 WARN [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data4/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:21,160 WARN [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data3/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:21,178 WARN [Thread-672 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:27:21,181 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x432caca874ad0958 with lease ID 0xa6dd06c378e0d825: Processing first storage report for DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe from datanode DatanodeRegistration(127.0.0.1:33771, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=38479, infoSecurePort=0, ipcPort=43589, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:21,181 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x432caca874ad0958 with lease ID 0xa6dd06c378e0d825: from storage DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe node DatanodeRegistration(127.0.0.1:33771, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=38479, infoSecurePort=0, ipcPort=43589, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T20:27:21,181 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x432caca874ad0958 with lease ID 0xa6dd06c378e0d825: Processing first storage report for DS-f3451470-18c7-466b-a8f6-02b842974553 from datanode DatanodeRegistration(127.0.0.1:33771, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=38479, infoSecurePort=0, ipcPort=43589, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:21,181 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x432caca874ad0958 with lease ID 0xa6dd06c378e0d825: from storage DS-f3451470-18c7-466b-a8f6-02b842974553 node DatanodeRegistration(127.0.0.1:33771, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=38479, infoSecurePort=0, ipcPort=43589, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:21,200 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394 2024-11-10T20:27:21,203 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/zookeeper_0, clientPort=55160, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T20:27:21,204 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55160 2024-11-10T20:27:21,204 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:21,206 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:21,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42605 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:27:21,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33771 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:27:21,217 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad with version=8 2024-11-10T20:27:21,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/hbase-staging 2024-11-10T20:27:21,219 INFO [Time-limited test {}] client.ConnectionUtils(128): master/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:27:21,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:21,219 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:21,220 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:27:21,220 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:21,220 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:27:21,220 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T20:27:21,220 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:27:21,221 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40057 2024-11-10T20:27:21,222 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40057 connecting to ZooKeeper ensemble=127.0.0.1:55160 2024-11-10T20:27:21,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400570x0, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:27:21,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40057-0x10193ddee5a0000 connected 2024-11-10T20:27:21,242 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:21,244 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:21,246 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:21,246 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad, hbase.cluster.distributed=false 2024-11-10T20:27:21,248 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:27:21,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40057 2024-11-10T20:27:21,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40057 2024-11-10T20:27:21,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40057 2024-11-10T20:27:21,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40057 2024-11-10T20:27:21,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40057 2024-11-10T20:27:21,266 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:27:21,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:21,267 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:21,267 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:27:21,267 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:21,267 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:27:21,267 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T20:27:21,267 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:27:21,268 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45329 2024-11-10T20:27:21,269 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45329 connecting to ZooKeeper ensemble=127.0.0.1:55160 2024-11-10T20:27:21,269 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:21,271 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:21,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453290x0, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:27:21,276 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45329-0x10193ddee5a0001 connected 2024-11-10T20:27:21,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:21,276 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T20:27:21,277 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T20:27:21,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T20:27:21,279 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:27:21,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45329 2024-11-10T20:27:21,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45329 2024-11-10T20:27:21,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45329 2024-11-10T20:27:21,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45329 2024-11-10T20:27:21,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45329 2024-11-10T20:27:21,285 INFO [regionserver/734d44df782b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:27:21,293 DEBUG [M:0;734d44df782b:40057 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;734d44df782b:40057 2024-11-10T20:27:21,293 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/734d44df782b,40057,1731270441219 2024-11-10T20:27:21,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:27:21,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:27:21,296 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/734d44df782b,40057,1731270441219 2024-11-10T20:27:21,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T20:27:21,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,298 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T20:27:21,298 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/734d44df782b,40057,1731270441219 from backup master directory 2024-11-10T20:27:21,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/734d44df782b,40057,1731270441219 2024-11-10T20:27:21,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:27:21,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:27:21,300 WARN [master/734d44df782b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:27:21,300 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=734d44df782b,40057,1731270441219 2024-11-10T20:27:21,305 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/hbase.id] with ID: 6c46e0ee-c9e0-43e3-a9ed-b9017cf8de69 2024-11-10T20:27:21,305 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/.tmp/hbase.id 2024-11-10T20:27:21,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42605 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:27:21,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33771 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:27:21,311 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/.tmp/hbase.id]:[hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/hbase.id] 2024-11-10T20:27:21,325 INFO [master/734d44df782b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:21,325 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T20:27:21,326 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T20:27:21,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33771 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:27:21,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42605 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:27:21,336 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:27:21,337 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T20:27:21,337 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:27:21,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33771 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:27:21,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42605 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:27:21,346 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store 2024-11-10T20:27:21,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33771 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:27:21,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42605 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:27:21,353 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:21,353 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:27:21,353 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:21,353 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:21,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:27:21,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:21,354 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:27:21,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270441353Disabling compacts and flushes for region at 1731270441353Disabling writes for close at 1731270441354 (+1 ms)Writing region close event to WAL at 1731270441354Closed at 1731270441354 2024-11-10T20:27:21,355 WARN [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/.initializing 2024-11-10T20:27:21,355 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219 2024-11-10T20:27:21,358 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C40057%2C1731270441219, suffix=, logDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219, archiveDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/oldWALs, maxLogs=10 2024-11-10T20:27:21,358 INFO [master/734d44df782b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40057%2C1731270441219.1731270441358 2024-11-10T20:27:21,363 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270441358 2024-11-10T20:27:21,366 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38479:38479),(127.0.0.1/127.0.0.1:40605:40605)] 2024-11-10T20:27:21,367 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:27:21,367 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:21,367 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,367 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,369 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,370 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T20:27:21,371 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:21,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:21,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T20:27:21,373 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:21,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:27:21,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T20:27:21,375 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:21,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:27:21,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,376 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T20:27:21,376 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:21,377 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:27:21,377 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,378 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,378 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,380 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,380 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,380 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T20:27:21,381 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:27:21,384 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:27:21,384 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=707721, jitterRate=-0.10008665919303894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T20:27:21,385 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731270441368Initializing all the Stores at 1731270441369 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270441369Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270441369Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270441369Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270441369Cleaning up temporary data from old regions at 1731270441380 (+11 ms)Region opened successfully at 1731270441385 (+5 ms) 2024-11-10T20:27:21,385 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T20:27:21,389 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2c37dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:27:21,390 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T20:27:21,390 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T20:27:21,390 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T20:27:21,390 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T20:27:21,390 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T20:27:21,391 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T20:27:21,391 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T20:27:21,393 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T20:27:21,394 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T20:27:21,395 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T20:27:21,395 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T20:27:21,396 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T20:27:21,398 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T20:27:21,398 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T20:27:21,399 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T20:27:21,400 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T20:27:21,401 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T20:27:21,402 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T20:27:21,404 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T20:27:21,405 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T20:27:21,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:21,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:21,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,408 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=734d44df782b,40057,1731270441219, sessionid=0x10193ddee5a0000, setting cluster-up flag (Was=false) 2024-11-10T20:27:21,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,415 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T20:27:21,417 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,40057,1731270441219 2024-11-10T20:27:21,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,425 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T20:27:21,426 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,40057,1731270441219 2024-11-10T20:27:21,428 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T20:27:21,430 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T20:27:21,430 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T20:27:21,430 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T20:27:21,430 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 734d44df782b,40057,1731270441219 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T20:27:21,431 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:27:21,432 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:27:21,432 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:27:21,432 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:27:21,432 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/734d44df782b:0, corePoolSize=10, maxPoolSize=10 2024-11-10T20:27:21,432 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,432 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:27:21,432 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,433 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731270471433 2024-11-10T20:27:21,433 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T20:27:21,433 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T20:27:21,433 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T20:27:21,433 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T20:27:21,433 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T20:27:21,433 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T20:27:21,433 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,434 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T20:27:21,434 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:27:21,434 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T20:27:21,434 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T20:27:21,434 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T20:27:21,434 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T20:27:21,434 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T20:27:21,434 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270441434,5,FailOnTimeoutGroup] 2024-11-10T20:27:21,435 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270441435,5,FailOnTimeoutGroup] 2024-11-10T20:27:21,435 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,435 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T20:27:21,435 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,435 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:21,435 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,435 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T20:27:21,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33771 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:27:21,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42605 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:27:21,444 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T20:27:21,444 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad 2024-11-10T20:27:21,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33771 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:27:21,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42605 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:27:21,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:21,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:27:21,455 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:27:21,455 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:21,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:21,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:27:21,457 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:27:21,457 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:21,458 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:21,458 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:27:21,459 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:27:21,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:21,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:21,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:27:21,461 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:27:21,461 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:21,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:21,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:27:21,463 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740 2024-11-10T20:27:21,463 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740 2024-11-10T20:27:21,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:27:21,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:27:21,465 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:27:21,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:27:21,468 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:27:21,469 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=791641, jitterRate=0.006623610854148865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:27:21,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731270441452Initializing all the Stores at 1731270441453 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270441453Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270441453Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270441453Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270441454 (+1 ms)Cleaning up temporary data from old regions at 1731270441465 (+11 ms)Region opened successfully at 1731270441470 (+5 ms) 2024-11-10T20:27:21,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:27:21,470 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:27:21,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:27:21,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:27:21,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:27:21,471 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:27:21,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270441470Disabling compacts and flushes for region at 1731270441470Disabling writes for close at 1731270441470Writing region close event to WAL at 1731270441471 (+1 ms)Closed at 1731270441471 2024-11-10T20:27:21,473 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:27:21,473 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T20:27:21,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T20:27:21,475 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:27:21,476 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T20:27:21,482 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(746): ClusterId : 6c46e0ee-c9e0-43e3-a9ed-b9017cf8de69 2024-11-10T20:27:21,482 DEBUG [RS:0;734d44df782b:45329 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T20:27:21,484 DEBUG [RS:0;734d44df782b:45329 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T20:27:21,484 DEBUG [RS:0;734d44df782b:45329 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T20:27:21,486 DEBUG [RS:0;734d44df782b:45329 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T20:27:21,486 DEBUG [RS:0;734d44df782b:45329 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa1ed50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:27:21,499 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;734d44df782b:45329 2024-11-10T20:27:21,499 INFO [RS:0;734d44df782b:45329 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T20:27:21,499 INFO [RS:0;734d44df782b:45329 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T20:27:21,499 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T20:27:21,500 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(2659): reportForDuty to master=734d44df782b,40057,1731270441219 with port=45329, startcode=1731270441266 2024-11-10T20:27:21,500 DEBUG [RS:0;734d44df782b:45329 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T20:27:21,502 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33677, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T20:27:21,503 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 734d44df782b,45329,1731270441266 2024-11-10T20:27:21,503 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40057 {}] master.ServerManager(517): Registering regionserver=734d44df782b,45329,1731270441266 2024-11-10T20:27:21,505 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad 2024-11-10T20:27:21,505 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34715 2024-11-10T20:27:21,505 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T20:27:21,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:27:21,507 DEBUG [RS:0;734d44df782b:45329 {}] zookeeper.ZKUtil(111): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/734d44df782b,45329,1731270441266 2024-11-10T20:27:21,507 WARN [RS:0;734d44df782b:45329 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:27:21,508 INFO [RS:0;734d44df782b:45329 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:27:21,508 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [734d44df782b,45329,1731270441266] 2024-11-10T20:27:21,508 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266 2024-11-10T20:27:21,511 INFO [RS:0;734d44df782b:45329 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T20:27:21,514 INFO [RS:0;734d44df782b:45329 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T20:27:21,514 INFO [RS:0;734d44df782b:45329 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T20:27:21,514 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,514 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T20:27:21,515 INFO [RS:0;734d44df782b:45329 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T20:27:21,515 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:27:21,516 DEBUG [RS:0;734d44df782b:45329 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:27:21,520 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,520 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,520 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,520 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,520 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,520 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,45329,1731270441266-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:27:21,537 INFO [RS:0;734d44df782b:45329 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T20:27:21,537 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,45329,1731270441266-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,538 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,538 INFO [RS:0;734d44df782b:45329 {}] regionserver.Replication(171): 734d44df782b,45329,1731270441266 started 2024-11-10T20:27:21,553 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:21,553 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1482): Serving as 734d44df782b,45329,1731270441266, RpcServer on 734d44df782b/172.17.0.2:45329, sessionid=0x10193ddee5a0001 2024-11-10T20:27:21,554 DEBUG [RS:0;734d44df782b:45329 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T20:27:21,554 DEBUG [RS:0;734d44df782b:45329 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 734d44df782b,45329,1731270441266 2024-11-10T20:27:21,554 DEBUG [RS:0;734d44df782b:45329 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,45329,1731270441266' 2024-11-10T20:27:21,554 DEBUG [RS:0;734d44df782b:45329 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T20:27:21,555 DEBUG [RS:0;734d44df782b:45329 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T20:27:21,555 DEBUG [RS:0;734d44df782b:45329 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T20:27:21,555 DEBUG [RS:0;734d44df782b:45329 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T20:27:21,555 DEBUG [RS:0;734d44df782b:45329 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 734d44df782b,45329,1731270441266 2024-11-10T20:27:21,555 DEBUG [RS:0;734d44df782b:45329 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,45329,1731270441266' 2024-11-10T20:27:21,555 DEBUG [RS:0;734d44df782b:45329 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T20:27:21,556 DEBUG [RS:0;734d44df782b:45329 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T20:27:21,557 DEBUG [RS:0;734d44df782b:45329 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T20:27:21,557 INFO [RS:0;734d44df782b:45329 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T20:27:21,557 INFO [RS:0;734d44df782b:45329 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T20:27:21,626 WARN [734d44df782b:40057 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T20:27:21,659 INFO [RS:0;734d44df782b:45329 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C45329%2C1731270441266, suffix=, logDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266, archiveDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs, maxLogs=32 2024-11-10T20:27:21,661 INFO [RS:0;734d44df782b:45329 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C45329%2C1731270441266.1731270441661 2024-11-10T20:27:21,668 INFO [RS:0;734d44df782b:45329 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 2024-11-10T20:27:21,696 DEBUG [RS:0;734d44df782b:45329 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38479:38479),(127.0.0.1/127.0.0.1:40605:40605)] 2024-11-10T20:27:21,876 DEBUG [734d44df782b:40057 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T20:27:21,877 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=734d44df782b,45329,1731270441266 2024-11-10T20:27:21,879 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,45329,1731270441266, state=OPENING 2024-11-10T20:27:21,880 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T20:27:21,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:21,883 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:27:21,883 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:27:21,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,45329,1731270441266}] 2024-11-10T20:27:21,883 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:27:22,036 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T20:27:22,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59365, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T20:27:22,042 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T20:27:22,043 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:27:22,045 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C45329%2C1731270441266.meta, suffix=.meta, logDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266, archiveDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs, maxLogs=32 2024-11-10T20:27:22,046 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta 2024-11-10T20:27:22,051 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta 2024-11-10T20:27:22,052 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38479:38479),(127.0.0.1/127.0.0.1:40605:40605)] 2024-11-10T20:27:22,053 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:27:22,053 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T20:27:22,053 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T20:27:22,053 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T20:27:22,053 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T20:27:22,053 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:22,053 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T20:27:22,054 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T20:27:22,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:27:22,056 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:27:22,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:22,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:22,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:27:22,057 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:27:22,058 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:22,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:22,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:27:22,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:27:22,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:22,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:22,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:27:22,060 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:27:22,060 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:22,061 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:27:22,061 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:27:22,062 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740 2024-11-10T20:27:22,063 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740 2024-11-10T20:27:22,064 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:27:22,064 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:27:22,065 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:27:22,066 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:27:22,067 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=776714, jitterRate=-0.012357980012893677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:27:22,067 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T20:27:22,067 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731270442054Writing region info on filesystem at 1731270442054Initializing all the Stores at 1731270442054Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270442055 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270442055Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270442055Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270442055Cleaning up temporary data from old regions at 1731270442064 (+9 ms)Running coprocessor post-open hooks at 1731270442067 (+3 ms)Region opened successfully at 1731270442067 2024-11-10T20:27:22,068 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731270442036 2024-11-10T20:27:22,071 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T20:27:22,071 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T20:27:22,072 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,45329,1731270441266 2024-11-10T20:27:22,073 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,45329,1731270441266, state=OPEN 2024-11-10T20:27:22,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:27:22,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:27:22,078 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=734d44df782b,45329,1731270441266 2024-11-10T20:27:22,078 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:27:22,078 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:27:22,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T20:27:22,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,45329,1731270441266 in 195 msec 2024-11-10T20:27:22,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T20:27:22,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-11-10T20:27:22,085 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:27:22,085 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T20:27:22,086 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:27:22,086 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,45329,1731270441266, seqNum=-1] 2024-11-10T20:27:22,087 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:27:22,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42665, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:27:22,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 664 msec 2024-11-10T20:27:22,094 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731270442094, completionTime=-1 2024-11-10T20:27:22,094 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T20:27:22,094 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T20:27:22,096 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T20:27:22,096 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731270502096 2024-11-10T20:27:22,096 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731270562096 2024-11-10T20:27:22,096 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-10T20:27:22,097 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,40057,1731270441219-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,097 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,40057,1731270441219-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,097 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,40057,1731270441219-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,097 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-734d44df782b:40057, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,097 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,097 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,099 DEBUG [master/734d44df782b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T20:27:22,101 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.801sec 2024-11-10T20:27:22,101 INFO [master/734d44df782b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T20:27:22,101 INFO [master/734d44df782b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T20:27:22,101 INFO [master/734d44df782b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T20:27:22,101 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T20:27:22,101 INFO [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T20:27:22,101 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,40057,1731270441219-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:27:22,102 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,40057,1731270441219-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T20:27:22,104 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T20:27:22,104 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T20:27:22,104 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,40057,1731270441219-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c8d161, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:27:22,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 734d44df782b,40057,-1 for getting cluster id 2024-11-10T20:27:22,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T20:27:22,197 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6c46e0ee-c9e0-43e3-a9ed-b9017cf8de69' 2024-11-10T20:27:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T20:27:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6c46e0ee-c9e0-43e3-a9ed-b9017cf8de69" 2024-11-10T20:27:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f45fe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:27:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [734d44df782b,40057,-1] 2024-11-10T20:27:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T20:27:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:22,200 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T20:27:22,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cae84f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:27:22,201 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:27:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,45329,1731270441266, seqNum=-1] 2024-11-10T20:27:22,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:27:22,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:27:22,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=734d44df782b,40057,1731270441219 2024-11-10T20:27:22,209 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:22,211 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T20:27:22,227 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:27:22,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:22,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:22,227 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:27:22,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:27:22,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:27:22,227 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T20:27:22,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:27:22,228 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43225 2024-11-10T20:27:22,229 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43225 connecting to ZooKeeper ensemble=127.0.0.1:55160 2024-11-10T20:27:22,230 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:22,231 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:27:22,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432250x0, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:27:22,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43225-0x10193ddee5a0002 connected 2024-11-10T20:27:22,236 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-10T20:27:22,236 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-10T20:27:22,237 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T20:27:22,240 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T20:27:22,241 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T20:27:22,242 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:27:22,243 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43225 2024-11-10T20:27:22,243 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43225 2024-11-10T20:27:22,243 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43225 2024-11-10T20:27:22,247 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43225 2024-11-10T20:27:22,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43225 2024-11-10T20:27:22,249 INFO [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(746): ClusterId : 6c46e0ee-c9e0-43e3-a9ed-b9017cf8de69 2024-11-10T20:27:22,249 DEBUG [RS:1;734d44df782b:43225 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T20:27:22,251 DEBUG [RS:1;734d44df782b:43225 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T20:27:22,251 DEBUG [RS:1;734d44df782b:43225 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T20:27:22,253 DEBUG [RS:1;734d44df782b:43225 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T20:27:22,253 DEBUG [RS:1;734d44df782b:43225 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700e07f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:27:22,265 DEBUG [RS:1;734d44df782b:43225 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;734d44df782b:43225 2024-11-10T20:27:22,265 INFO [RS:1;734d44df782b:43225 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T20:27:22,265 INFO [RS:1;734d44df782b:43225 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T20:27:22,265 DEBUG [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T20:27:22,266 INFO [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(2659): reportForDuty to master=734d44df782b,40057,1731270441219 with port=43225, startcode=1731270442226 2024-11-10T20:27:22,266 DEBUG [RS:1;734d44df782b:43225 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T20:27:22,268 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37493, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T20:27:22,268 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 734d44df782b,43225,1731270442226 2024-11-10T20:27:22,268 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40057 {}] master.ServerManager(517): Registering regionserver=734d44df782b,43225,1731270442226 2024-11-10T20:27:22,270 DEBUG [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad 2024-11-10T20:27:22,270 DEBUG [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34715 2024-11-10T20:27:22,270 DEBUG [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T20:27:22,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:27:22,273 DEBUG [RS:1;734d44df782b:43225 {}] zookeeper.ZKUtil(111): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/734d44df782b,43225,1731270442226 2024-11-10T20:27:22,273 WARN [RS:1;734d44df782b:43225 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:27:22,273 INFO [RS:1;734d44df782b:43225 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:27:22,273 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [734d44df782b,43225,1731270442226] 2024-11-10T20:27:22,273 DEBUG [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226 2024-11-10T20:27:22,276 INFO [RS:1;734d44df782b:43225 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T20:27:22,278 INFO [RS:1;734d44df782b:43225 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T20:27:22,279 INFO [RS:1;734d44df782b:43225 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T20:27:22,279 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,279 INFO [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T20:27:22,280 INFO [RS:1;734d44df782b:43225 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T20:27:22,280 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,280 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,280 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,280 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,280 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,280 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,280 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:27:22,281 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,281 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,281 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,281 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,281 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,281 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:27:22,281 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:27:22,281 DEBUG [RS:1;734d44df782b:43225 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:27:22,281 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,281 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,281 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,281 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,281 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,281 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43225,1731270442226-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:27:22,297 INFO [RS:1;734d44df782b:43225 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T20:27:22,297 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43225,1731270442226-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,297 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,297 INFO [RS:1;734d44df782b:43225 {}] regionserver.Replication(171): 734d44df782b,43225,1731270442226 started 2024-11-10T20:27:22,311 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:27:22,311 INFO [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(1482): Serving as 734d44df782b,43225,1731270442226, RpcServer on 734d44df782b/172.17.0.2:43225, sessionid=0x10193ddee5a0002 2024-11-10T20:27:22,311 DEBUG [RS:1;734d44df782b:43225 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T20:27:22,311 DEBUG [RS:1;734d44df782b:43225 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 734d44df782b,43225,1731270442226 2024-11-10T20:27:22,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;734d44df782b:43225,5,FailOnTimeoutGroup] 2024-11-10T20:27:22,311 DEBUG [RS:1;734d44df782b:43225 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,43225,1731270442226' 2024-11-10T20:27:22,311 DEBUG [RS:1;734d44df782b:43225 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T20:27:22,311 INFO [Time-limited test {}] wal.TestLogRolling(207): Replication=2 2024-11-10T20:27:22,312 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T20:27:22,312 DEBUG [RS:1;734d44df782b:43225 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T20:27:22,312 DEBUG [RS:1;734d44df782b:43225 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T20:27:22,312 DEBUG [RS:1;734d44df782b:43225 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T20:27:22,312 DEBUG [RS:1;734d44df782b:43225 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 734d44df782b,43225,1731270442226 2024-11-10T20:27:22,312 DEBUG [RS:1;734d44df782b:43225 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,43225,1731270442226' 2024-11-10T20:27:22,312 DEBUG [RS:1;734d44df782b:43225 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T20:27:22,313 DEBUG [RS:1;734d44df782b:43225 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T20:27:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 734d44df782b,40057,1731270441219 2024-11-10T20:27:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21b2f882 2024-11-10T20:27:22,313 DEBUG [RS:1;734d44df782b:43225 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T20:27:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T20:27:22,313 INFO [RS:1;734d44df782b:43225 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T20:27:22,313 INFO [RS:1;734d44df782b:43225 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T20:27:22,315 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T20:27:22,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40057 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T20:27:22,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T20:27:22,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40057 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:27:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40057 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T20:27:22,319 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T20:27:22,319 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:22,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40057 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-11-10T20:27:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:27:22,320 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T20:27:22,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42605 is added to blk_1073741835_1011 (size=393) 2024-11-10T20:27:22,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33771 is added to blk_1073741835_1011 (size=393) 2024-11-10T20:27:22,329 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d422e1883575bc1a2891d04cf7083b0b, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad 2024-11-10T20:27:22,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42605 is added to blk_1073741836_1012 (size=76) 2024-11-10T20:27:22,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33771 is added to blk_1073741836_1012 (size=76) 2024-11-10T20:27:22,336 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:22,337 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing d422e1883575bc1a2891d04cf7083b0b, disabling compactions & flushes 2024-11-10T20:27:22,337 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:22,337 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:22,337 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. after waiting 0 ms 2024-11-10T20:27:22,337 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:22,337 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:22,337 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for d422e1883575bc1a2891d04cf7083b0b: Waiting for close lock at 1731270442336Disabling compacts and flushes for region at 1731270442336Disabling writes for close at 1731270442337 (+1 ms)Writing region close event to WAL at 1731270442337Closed at 1731270442337 2024-11-10T20:27:22,338 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T20:27:22,338 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731270442338"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731270442338"}]},"ts":"1731270442338"} 2024-11-10T20:27:22,341 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T20:27:22,342 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T20:27:22,342 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270442342"}]},"ts":"1731270442342"} 2024-11-10T20:27:22,345 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-11-10T20:27:22,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d422e1883575bc1a2891d04cf7083b0b, ASSIGN}] 2024-11-10T20:27:22,346 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d422e1883575bc1a2891d04cf7083b0b, ASSIGN 2024-11-10T20:27:22,347 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d422e1883575bc1a2891d04cf7083b0b, ASSIGN; state=OFFLINE, location=734d44df782b,45329,1731270441266; forceNewPlan=false, retain=false 2024-11-10T20:27:22,415 INFO [RS:1;734d44df782b:43225 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C43225%2C1731270442226, suffix=, logDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226, archiveDir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs, maxLogs=32 2024-11-10T20:27:22,416 INFO [RS:1;734d44df782b:43225 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C43225%2C1731270442226.1731270442416 2024-11-10T20:27:22,422 INFO [RS:1;734d44df782b:43225 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 2024-11-10T20:27:22,423 DEBUG [RS:1;734d44df782b:43225 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40605:40605),(127.0.0.1/127.0.0.1:38479:38479)] 2024-11-10T20:27:22,498 INFO [734d44df782b:40057 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T20:27:22,499 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d422e1883575bc1a2891d04cf7083b0b, regionState=OPENING, regionLocation=734d44df782b,45329,1731270441266 2024-11-10T20:27:22,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d422e1883575bc1a2891d04cf7083b0b, ASSIGN because future has completed 2024-11-10T20:27:22,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d422e1883575bc1a2891d04cf7083b0b, server=734d44df782b,45329,1731270441266}] 2024-11-10T20:27:22,659 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:22,660 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d422e1883575bc1a2891d04cf7083b0b, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:27:22,660 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,660 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:27:22,660 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,660 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,662 INFO [StoreOpener-d422e1883575bc1a2891d04cf7083b0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,663 INFO [StoreOpener-d422e1883575bc1a2891d04cf7083b0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d422e1883575bc1a2891d04cf7083b0b columnFamilyName info 2024-11-10T20:27:22,664 DEBUG [StoreOpener-d422e1883575bc1a2891d04cf7083b0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:27:22,664 INFO [StoreOpener-d422e1883575bc1a2891d04cf7083b0b-1 {}] regionserver.HStore(327): Store=d422e1883575bc1a2891d04cf7083b0b/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:27:22,664 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,665 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,665 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,666 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,666 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,667 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,669 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:27:22,669 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d422e1883575bc1a2891d04cf7083b0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=792818, jitterRate=0.008120566606521606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T20:27:22,670 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:22,670 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d422e1883575bc1a2891d04cf7083b0b: Running coprocessor pre-open hook at 1731270442661Writing region info on filesystem at 1731270442661Initializing all the Stores at 1731270442662 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270442662Cleaning up temporary data from old regions at 1731270442666 (+4 ms)Running coprocessor post-open hooks at 1731270442670 (+4 ms)Region opened successfully at 1731270442670 2024-11-10T20:27:22,671 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b., pid=6, masterSystemTime=1731270442655 2024-11-10T20:27:22,674 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:22,674 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:22,675 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d422e1883575bc1a2891d04cf7083b0b, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,45329,1731270441266 2024-11-10T20:27:22,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d422e1883575bc1a2891d04cf7083b0b, server=734d44df782b,45329,1731270441266 because future has completed 2024-11-10T20:27:22,679 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:22,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T20:27:22,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d422e1883575bc1a2891d04cf7083b0b, server=734d44df782b,45329,1731270441266 in 177 msec 2024-11-10T20:27:22,683 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:22,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T20:27:22,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=d422e1883575bc1a2891d04cf7083b0b, ASSIGN in 337 msec 2024-11-10T20:27:22,686 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T20:27:22,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270442686"}]},"ts":"1731270442686"} 2024-11-10T20:27:22,689 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-11-10T20:27:22,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T20:27:22,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 375 msec 2024-11-10T20:27:23,197 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T20:27:23,198 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:23,217 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:23,217 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:23,218 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:23,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T20:27:23,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-11-10T20:27:23,226 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-10T20:27:27,511 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T20:27:27,512 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-11-10T20:27:28,162 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T20:27:28,164 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:28,181 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:28,185 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:28,185 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:27:32,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:27:32,342 INFO [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-11-10T20:27:32,342 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-11-10T20:27:32,345 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T20:27:32,345 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:32,357 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:32,360 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:32,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:32,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:32,361 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:27:32,362 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@21ddd8f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:32,362 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d0f4a9a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:32,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@730725ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/java.io.tmpdir/jetty-localhost-39095-hadoop-hdfs-3_4_1-tests_jar-_-any-3893235082293554470/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:32,477 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@20e1b523{HTTP/1.1, (http/1.1)}{localhost:39095} 2024-11-10T20:27:32,477 INFO [Time-limited test {}] server.Server(415): Started @114951ms 2024-11-10T20:27:32,478 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:27:32,509 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:32,512 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:32,516 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:32,516 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:32,516 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:27:32,517 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@83c4e47{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:32,518 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@519de6b2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:32,574 WARN [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data5/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:32,574 WARN [Thread-830 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data6/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:32,595 WARN [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:27:32,597 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x49088437ab5b4bdd with lease ID 0xa6dd06c378e0d826: Processing first storage report for DS-ab0f1e07-554d-4c96-804e-8417d9c98653 from datanode DatanodeRegistration(127.0.0.1:34905, datanodeUuid=41d0d3a5-1f5d-4366-8706-bfc81c648461, infoPort=45307, infoSecurePort=0, ipcPort=41275, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x49088437ab5b4bdd with lease ID 0xa6dd06c378e0d826: from storage DS-ab0f1e07-554d-4c96-804e-8417d9c98653 node DatanodeRegistration(127.0.0.1:34905, datanodeUuid=41d0d3a5-1f5d-4366-8706-bfc81c648461, infoPort=45307, infoSecurePort=0, ipcPort=41275, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x49088437ab5b4bdd with lease ID 0xa6dd06c378e0d826: Processing first storage report for DS-3c39cfb9-25ab-4af3-a668-b5120c926992 from datanode DatanodeRegistration(127.0.0.1:34905, datanodeUuid=41d0d3a5-1f5d-4366-8706-bfc81c648461, infoPort=45307, infoSecurePort=0, ipcPort=41275, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x49088437ab5b4bdd with lease ID 0xa6dd06c378e0d826: from storage DS-3c39cfb9-25ab-4af3-a668-b5120c926992 node DatanodeRegistration(127.0.0.1:34905, datanodeUuid=41d0d3a5-1f5d-4366-8706-bfc81c648461, infoPort=45307, infoSecurePort=0, ipcPort=41275, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:32,646 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4438143d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/java.io.tmpdir/jetty-localhost-46551-hadoop-hdfs-3_4_1-tests_jar-_-any-4939927093208111317/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:32,647 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32dac098{HTTP/1.1, (http/1.1)}{localhost:46551} 2024-11-10T20:27:32,647 INFO [Time-limited test {}] server.Server(415): Started @115121ms 2024-11-10T20:27:32,649 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:27:32,680 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:32,683 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:32,684 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:32,684 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:32,684 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:27:32,685 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3dc0bdb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:32,685 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50ff2063{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:32,730 WARN [Thread-864 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:32,730 WARN [Thread-865 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:32,747 WARN [Thread-844 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:27:32,750 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5cf1cc5f0a04366d with lease ID 0xa6dd06c378e0d827: Processing first storage report for DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8 from datanode DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:32,750 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5cf1cc5f0a04366d with lease ID 0xa6dd06c378e0d827: from storage DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8 node DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:32,750 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5cf1cc5f0a04366d with lease ID 0xa6dd06c378e0d827: Processing first storage report for DS-d07d16b9-9f89-4ac3-a894-d96f101a74c7 from datanode DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:32,750 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5cf1cc5f0a04366d with lease ID 0xa6dd06c378e0d827: from storage DS-d07d16b9-9f89-4ac3-a894-d96f101a74c7 node DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:32,803 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c81b75d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/java.io.tmpdir/jetty-localhost-42343-hadoop-hdfs-3_4_1-tests_jar-_-any-1367884222211509794/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:32,804 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f55aa3b{HTTP/1.1, (http/1.1)}{localhost:42343} 2024-11-10T20:27:32,804 INFO [Time-limited test {}] server.Server(415): Started @115278ms 2024-11-10T20:27:32,805 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:27:32,883 WARN [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data9/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:32,883 WARN [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data10/current/BP-1984011214-172.17.0.2-1731270440526/current, will proceed with Du for space computation calculation, 2024-11-10T20:27:32,900 WARN [Thread-879 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:27:32,902 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x429eba9e05ccc6af with lease ID 0xa6dd06c378e0d828: Processing first storage report for DS-fd121265-d2bb-4b39-9dac-db7030c4a90c from datanode DatanodeRegistration(127.0.0.1:37785, datanodeUuid=834473bd-34f2-498d-b446-3ef6ebb84ff7, infoPort=43999, infoSecurePort=0, ipcPort=43917, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:32,902 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x429eba9e05ccc6af with lease ID 0xa6dd06c378e0d828: from storage DS-fd121265-d2bb-4b39-9dac-db7030c4a90c node DatanodeRegistration(127.0.0.1:37785, datanodeUuid=834473bd-34f2-498d-b446-3ef6ebb84ff7, infoPort=43999, infoSecurePort=0, ipcPort=43917, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:32,902 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x429eba9e05ccc6af with lease ID 0xa6dd06c378e0d828: Processing first storage report for DS-bf636108-8a6d-409b-a234-6f93a2bebf60 from datanode DatanodeRegistration(127.0.0.1:37785, datanodeUuid=834473bd-34f2-498d-b446-3ef6ebb84ff7, infoPort=43999, infoSecurePort=0, ipcPort=43917, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526) 2024-11-10T20:27:32,902 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x429eba9e05ccc6af with lease ID 0xa6dd06c378e0d828: from storage DS-bf636108-8a6d-409b-a234-6f93a2bebf60 node DatanodeRegistration(127.0.0.1:37785, datanodeUuid=834473bd-34f2-498d-b446-3ef6ebb84ff7, infoPort=43999, infoSecurePort=0, ipcPort=43917, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:32,924 WARN [ResponseProcessor for block BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,924 WARN [ResponseProcessor for block BP-1984011214-172.17.0.2-1731270440526:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1984011214-172.17.0.2-1731270440526:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,924 WARN [ResponseProcessor for block BP-1984011214-172.17.0.2-1731270440526:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1984011214-172.17.0.2-1731270440526:blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,925 WARN [ResponseProcessor for block BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013 java.io.IOException: Bad response ERROR for BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,925 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 block BP-1984011214-172.17.0.2-1731270440526:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:32,925 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta block BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:32,925 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 block BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK], DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:32,925 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270441358 block BP-1984011214-172.17.0.2-1731270440526:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:32,925 WARN [PacketResponder: BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33771] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:32,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:44568 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33771:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44568 dst: /127.0.0.1:33771 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:32,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:33786 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42605:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33786 dst: /127.0.0.1:42605 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:32,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1772045698_22 at /127.0.0.1:44610 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:33771:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44610 dst: /127.0.0.1:33771 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:32,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1101538459_22 at /127.0.0.1:44520 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33771:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44520 dst: /127.0.0.1:33771 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:32,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f2859b3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:32,926 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1772045698_22 at /127.0.0.1:33804 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:42605:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33804 dst: /127.0.0.1:42605 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:32,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:44558 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33771:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44558 dst: /127.0.0.1:33771 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:32,927 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:33782 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42605:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33782 dst: /127.0.0.1:42605 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:32,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1101538459_22 at /127.0.0.1:33762 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42605:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33762 dst: /127.0.0.1:42605 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:32,928 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3b918d2a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:32,928 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:32,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7eee535{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:32,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@66182b08{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:32,929 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:27:32,930 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1984011214-172.17.0.2-1731270440526 (Datanode Uuid 422d81f1-121b-44ef-a243-9c418ef1c7dc) service to localhost/127.0.0.1:34715 2024-11-10T20:27:32,930 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data3/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:32,930 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:27:32,931 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:27:32,931 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:27:32,931 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data4/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:32,932 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 block BP-1984011214-172.17.0.2-1731270440526:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,932 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta block BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,932 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 block BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,932 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270441358 block BP-1984011214-172.17.0.2-1731270440526:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,934 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6de86657{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:32,934 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6787773a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:32,934 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:32,934 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a107105{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:32,934 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@65dec1b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:32,936 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:27:32,936 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:27:32,936 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1984011214-172.17.0.2-1731270440526 (Datanode Uuid 8ba45455-c138-4c8c-bb3f-c05acbb2cf33) service to localhost/127.0.0.1:34715 2024-11-10T20:27:32,936 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:27:32,936 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data1/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:32,937 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data2/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:32,937 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:27:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b., hostname=734d44df782b,45329,1731270441266, seqNum=2] 2024-11-10T20:27:32,942 ERROR [FSHLog-0-hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad-prefix:734d44df782b,45329,1731270441266 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,942 WARN [FSHLog-0-hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad-prefix:734d44df782b,45329,1731270441266 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,942 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,942 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C45329%2C1731270441266:(num 1731270441661) roll requested 2024-11-10T20:27:32,943 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C45329%2C1731270441266.1731270452943 2024-11-10T20:27:32,945 WARN [Thread-901 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,946 WARN [Thread-901 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:32,946 WARN [Thread-901 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741838_1018 2024-11-10T20:27:32,948 WARN [Thread-901 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] 2024-11-10T20:27:32,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:32,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:32,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:32,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:32,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:32,955 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270452943 2024-11-10T20:27:32,955 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,955 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:32,956 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45307:45307),(127.0.0.1/127.0.0.1:43999:43999)] 2024-11-10T20:27:32,956 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 is not closed yet, will try archiving it next time 2024-11-10T20:27:32,956 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-10T20:27:32,957 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-10T20:27:32,957 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 2024-11-10T20:27:32,960 WARN [IPC Server handler 1 on default port 34715 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741833_1009 2024-11-10T20:27:32,964 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 after 5ms 2024-11-10T20:27:33,338 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:34,282 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:34,956 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:34,958 INFO [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270452943 2024-11-10T20:27:34,958 WARN [ResponseProcessor for block BP-1984011214-172.17.0.2-1731270440526:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1984011214-172.17.0.2-1731270440526:blk_1073741839_1019 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:34,959 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270452943 block BP-1984011214-172.17.0.2-1731270440526:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]) is bad. 2024-11-10T20:27:34,959 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:33884 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:34905:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33884 dst: /127.0.0.1:34905 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:34,960 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:46458 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:37785:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46458 dst: /127.0.0.1:37785 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:34,961 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@730725ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:34,961 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@20e1b523{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:34,962 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:34,962 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d0f4a9a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:34,962 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@21ddd8f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:34,963 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:27:34,963 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:27:34,963 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1984011214-172.17.0.2-1731270440526 (Datanode Uuid 41d0d3a5-1f5d-4366-8706-bfc81c648461) service to localhost/127.0.0.1:34715 2024-11-10T20:27:34,963 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:27:34,963 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data5/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:34,963 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data6/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:34,964 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:27:35,338 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:36,282 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:36,957 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:36,957 WARN [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]] 2024-11-10T20:27:36,958 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C45329%2C1731270441266:(num 1731270452943) roll requested 2024-11-10T20:27:36,958 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C45329%2C1731270441266.1731270456958 2024-11-10T20:27:36,961 WARN [Thread-910 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:36,961 WARN [Thread-910 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:36,961 WARN [Thread-910 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741840_1022 2024-11-10T20:27:36,962 WARN [Thread-910 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] 2024-11-10T20:27:36,963 WARN [Thread-910 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:36,964 WARN [Thread-910 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]) is bad. 2024-11-10T20:27:36,964 WARN [Thread-910 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741841_1023 2024-11-10T20:27:36,964 WARN [Thread-910 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK] 2024-11-10T20:27:36,965 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 after 4008ms 2024-11-10T20:27:36,966 WARN [Thread-910 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34905 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:36,966 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34466 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741842_1024] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741842_1024 to mirror 127.0.0.1:34905 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:36,967 WARN [Thread-910 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]) is bad. 2024-11-10T20:27:36,967 WARN [Thread-910 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741842_1024 2024-11-10T20:27:36,967 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34466 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741842_1024] {}] datanode.BlockReceiver(316): Block 1073741842 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T20:27:36,967 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34466 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34466 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:36,967 WARN [Thread-910 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK] 2024-11-10T20:27:36,969 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T20:27:36,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:36,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:36,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:36,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:36,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:36,972 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270452943 with entries=3, filesize=3.51 KB; new WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270456958 2024-11-10T20:27:36,973 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43999:43999),(127.0.0.1/127.0.0.1:39841:39841)] 2024-11-10T20:27:36,973 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 is not closed yet, will try archiving it next time 2024-11-10T20:27:36,973 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270452943 is not closed yet, will try archiving it next time 2024-11-10T20:27:36,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37785 is added to blk_1073741839_1021 (size=3600) 2024-11-10T20:27:37,339 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:37,375 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 is not closed yet, will try archiving it next time 2024-11-10T20:27:38,283 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:38,913 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@61274e91[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37785, datanodeUuid=834473bd-34f2-498d-b446-3ef6ebb84ff7, infoPort=43999, infoSecurePort=0, ipcPort=43917, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741839_1021 to 127.0.0.1:34905 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:38,973 WARN [ResponseProcessor for block BP-1984011214-172.17.0.2-1731270440526:blk_1073741843_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1984011214-172.17.0.2-1731270440526:blk_1073741843_1025 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:38,973 WARN [DataStreamer for file /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270456958 block BP-1984011214-172.17.0.2-1731270440526:blk_1073741843_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:38,973 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:38,973 WARN [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]] 2024-11-10T20:27:38,974 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C45329%2C1731270441266:(num 1731270456958) roll requested 2024-11-10T20:27:38,973 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:46478 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:37785:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46478 dst: /127.0.0.1:37785 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:38,974 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34468 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34468 dst: /127.0.0.1:46141 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:38,974 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C45329%2C1731270441266.1731270458974 2024-11-10T20:27:38,975 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c81b75d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:38,976 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f55aa3b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:27:38,976 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:27:38,976 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@50ff2063{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:27:38,976 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3dc0bdb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,STOPPED} 2024-11-10T20:27:38,978 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:27:38,978 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:27:38,978 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:27:38,978 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1984011214-172.17.0.2-1731270440526 (Datanode Uuid 834473bd-34f2-498d-b446-3ef6ebb84ff7) service to localhost/127.0.0.1:34715 2024-11-10T20:27:38,978 WARN [Thread-920 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:38,978 WARN [Thread-920 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK], DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]) is bad. 2024-11-10T20:27:38,978 WARN [Thread-920 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741844_1027 2024-11-10T20:27:38,978 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data9/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:38,979 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data10/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:27:38,979 WARN [Thread-920 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK] 2024-11-10T20:27:38,979 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:27:38,980 WARN [Thread-920 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:38,981 WARN [Thread-920 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK], DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:38,981 WARN [Thread-920 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741845_1028 2024-11-10T20:27:38,982 WARN [Thread-920 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:38,983 WARN [Thread-920 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:38,983 WARN [Thread-920 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]) is bad. 2024-11-10T20:27:38,983 WARN [Thread-920 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741846_1029 2024-11-10T20:27:38,983 WARN [Thread-920 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK] 2024-11-10T20:27:38,986 WARN [Thread-920 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33771 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:38,986 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34486 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741847_1030 to mirror 127.0.0.1:33771 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:38,986 WARN [Thread-920 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:38,986 WARN [Thread-920 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741847_1030 2024-11-10T20:27:38,986 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34486 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T20:27:38,987 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34486 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34486 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:38,987 WARN [Thread-920 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] 2024-11-10T20:27:38,988 WARN [IPC Server handler 4 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T20:27:38,988 WARN [IPC Server handler 4 on default port 34715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T20:27:38,988 WARN [IPC Server handler 4 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T20:27:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45329 {}] regionserver.HRegion(8855): Flush requested on d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:38,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d422e1883575bc1a2891d04cf7083b0b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:27:38,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:38,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:38,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:38,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:38,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:38,992 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270456958 with entries=6, filesize=6.11 KB; new WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270458974 2024-11-10T20:27:38,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741843_1026 (size=6261) 2024-11-10T20:27:38,994 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 is not closed yet, will try archiving it next time 2024-11-10T20:27:38,996 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39841:39841)] 2024-11-10T20:27:38,996 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 is not closed yet, will try archiving it next time 2024-11-10T20:27:39,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/6ef522d6670c49e180aee6a40e50c26c is 1080, key is row0002/info:/1731270454965/Put/seqid=0 2024-11-10T20:27:39,015 WARN [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:39,015 WARN [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]) is bad. 2024-11-10T20:27:39,015 WARN [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741849_1032 2024-11-10T20:27:39,016 WARN [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK] 2024-11-10T20:27:39,017 WARN [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:39,017 WARN [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK], DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]) is bad. 2024-11-10T20:27:39,017 WARN [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741850_1033 2024-11-10T20:27:39,018 WARN [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK] 2024-11-10T20:27:39,020 WARN [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37785 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:39,020 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34496 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741851_1034] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741851_1034 to mirror 127.0.0.1:37785 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:39,020 WARN [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:39,020 WARN [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741851_1034 2024-11-10T20:27:39,020 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34496 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741851_1034] {}] datanode.BlockReceiver(316): Block 1073741851 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:39,020 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34496 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741851_1034] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34496 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:39,021 WARN [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:39,022 WARN [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:39,022 WARN [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:39,022 WARN [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741852_1035 2024-11-10T20:27:39,022 WARN [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] 2024-11-10T20:27:39,023 WARN [IPC Server handler 4 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T20:27:39,023 WARN [IPC Server handler 4 on default port 34715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T20:27:39,023 WARN [IPC Server handler 4 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T20:27:39,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741853_1036 (size=10347) 2024-11-10T20:27:39,339 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:39,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/6ef522d6670c49e180aee6a40e50c26c 2024-11-10T20:27:39,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/6ef522d6670c49e180aee6a40e50c26c as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/6ef522d6670c49e180aee6a40e50c26c 2024-11-10T20:27:39,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/6ef522d6670c49e180aee6a40e50c26c, entries=5, sequenceid=11, filesize=10.1 K 2024-11-10T20:27:39,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for d422e1883575bc1a2891d04cf7083b0b in 452ms, sequenceid=11, compaction requested=false 2024-11-10T20:27:39,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d422e1883575bc1a2891d04cf7083b0b: 2024-11-10T20:27:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45329 {}] regionserver.HRegion(8855): Flush requested on d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:39,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d422e1883575bc1a2891d04cf7083b0b 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-11-10T20:27:39,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/1641821d99914102b85d3d62bba5cb40 is 1080, key is row0007/info:/1731270458990/Put/seqid=0 2024-11-10T20:27:39,620 WARN [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:39,620 WARN [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:39,620 WARN [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741854_1037 2024-11-10T20:27:39,621 WARN [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] 2024-11-10T20:27:39,622 WARN [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:39,623 WARN [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK], DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:39,623 WARN [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741855_1038 2024-11-10T20:27:39,623 WARN [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:39,625 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34542 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741856_1039 to mirror 127.0.0.1:34905 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:39,625 WARN [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34905 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:39,626 WARN [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]) is bad. 2024-11-10T20:27:39,625 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34542 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:39,626 WARN [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741856_1039 2024-11-10T20:27:39,626 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34542 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34542 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:39,626 WARN [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK] 2024-11-10T20:27:39,628 WARN [Thread-932 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42605 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:39,628 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34546 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741857_1040 to mirror 127.0.0.1:42605 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:39,629 WARN [Thread-932 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]) is bad. 2024-11-10T20:27:39,629 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34546 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:39,629 WARN [Thread-932 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741857_1040 2024-11-10T20:27:39,629 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34546 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34546 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:39,629 WARN [Thread-932 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK] 2024-11-10T20:27:39,630 WARN [IPC Server handler 4 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T20:27:39,630 WARN [IPC Server handler 4 on default port 34715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T20:27:39,630 WARN [IPC Server handler 4 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T20:27:39,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741858_1041 (size=12506) 2024-11-10T20:27:40,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/1641821d99914102b85d3d62bba5cb40 2024-11-10T20:27:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/1641821d99914102b85d3d62bba5cb40 as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/1641821d99914102b85d3d62bba5cb40 2024-11-10T20:27:40,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/1641821d99914102b85d3d62bba5cb40, entries=7, sequenceid=24, filesize=12.2 K 2024-11-10T20:27:40,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for d422e1883575bc1a2891d04cf7083b0b in 433ms, sequenceid=24, compaction requested=false 2024-11-10T20:27:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d422e1883575bc1a2891d04cf7083b0b: 2024-11-10T20:27:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K 2024-11-10T20:27:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/1641821d99914102b85d3d62bba5cb40 because midkey is the same as first or last row 2024-11-10T20:27:40,283 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:40,996 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:40,996 WARN [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]] 2024-11-10T20:27:40,997 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C45329%2C1731270441266:(num 1731270458974) roll requested 2024-11-10T20:27:40,997 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C45329%2C1731270441266.1731270460997 2024-11-10T20:27:41,000 WARN [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,000 WARN [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]) is bad. 2024-11-10T20:27:41,000 WARN [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741859_1042 2024-11-10T20:27:41,001 WARN [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK] 2024-11-10T20:27:41,002 WARN [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,002 WARN [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:41,002 WARN [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741860_1043 2024-11-10T20:27:41,002 WARN [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:41,004 WARN [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,004 WARN [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK], DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]) is bad. 2024-11-10T20:27:41,004 WARN [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741861_1044 2024-11-10T20:27:41,004 WARN [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK] 2024-11-10T20:27:41,006 WARN [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,006 WARN [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:41,006 WARN [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741862_1045 2024-11-10T20:27:41,006 WARN [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] 2024-11-10T20:27:41,007 WARN [IPC Server handler 0 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T20:27:41,007 WARN [IPC Server handler 0 on default port 34715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T20:27:41,007 WARN [IPC Server handler 0 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T20:27:41,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:41,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:41,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:41,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:41,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:41,010 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270458974 with entries=18, filesize=18.21 KB; new WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270460997 2024-11-10T20:27:41,011 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39841:39841)] 2024-11-10T20:27:41,011 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 is not closed yet, will try archiving it next time 2024-11-10T20:27:41,011 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270458974 is not closed yet, will try archiving it next time 2024-11-10T20:27:41,011 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270452943 to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs/734d44df782b%2C45329%2C1731270441266.1731270452943 2024-11-10T20:27:41,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741848_1031 (size=18655) 2024-11-10T20:27:41,013 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270456958 to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs/734d44df782b%2C45329%2C1731270441266.1731270456958 2024-11-10T20:27:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45329 {}] regionserver.HRegion(8855): Flush requested on d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:41,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d422e1883575bc1a2891d04cf7083b0b 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T20:27:41,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/657fecc6b16841578af4e500528264c9 is 1079, key is tmprow/info:/1731270461031/Put/seqid=0 2024-11-10T20:27:41,040 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37785 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,039 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34420 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741864_1047] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741864_1047 to mirror 127.0.0.1:37785 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:41,040 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:41,040 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741864_1047 2024-11-10T20:27:41,040 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34420 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741864_1047] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:41,040 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34420 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741864_1047] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34420 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:41,040 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:41,042 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,042 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:41,042 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741865_1048 2024-11-10T20:27:41,042 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] 2024-11-10T20:27:41,043 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,043 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]) is bad. 2024-11-10T20:27:41,043 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741866_1049 2024-11-10T20:27:41,044 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK] 2024-11-10T20:27:41,045 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,045 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]) is bad. 2024-11-10T20:27:41,045 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741867_1050 2024-11-10T20:27:41,046 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK] 2024-11-10T20:27:41,046 WARN [IPC Server handler 2 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T20:27:41,046 WARN [IPC Server handler 2 on default port 34715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T20:27:41,046 WARN [IPC Server handler 2 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T20:27:41,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741868_1051 (size=6027) 2024-11-10T20:27:41,339 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,413 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 is not closed yet, will try archiving it next time 2024-11-10T20:27:41,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/657fecc6b16841578af4e500528264c9 2024-11-10T20:27:41,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/657fecc6b16841578af4e500528264c9 as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/657fecc6b16841578af4e500528264c9 2024-11-10T20:27:41,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/657fecc6b16841578af4e500528264c9, entries=1, sequenceid=34, filesize=5.9 K 2024-11-10T20:27:41,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for d422e1883575bc1a2891d04cf7083b0b in 433ms, sequenceid=34, compaction requested=true 2024-11-10T20:27:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d422e1883575bc1a2891d04cf7083b0b: 2024-11-10T20:27:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K 2024-11-10T20:27:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/1641821d99914102b85d3d62bba5cb40 because midkey is the same as first or last row 2024-11-10T20:27:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d422e1883575bc1a2891d04cf7083b0b:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:27:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:27:41,465 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:27:41,467 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:27:41,467 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HStore(1541): d422e1883575bc1a2891d04cf7083b0b/info is initiating minor compaction (all files) 2024-11-10T20:27:41,467 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d422e1883575bc1a2891d04cf7083b0b/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:41,467 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/6ef522d6670c49e180aee6a40e50c26c, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/1641821d99914102b85d3d62bba5cb40, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/657fecc6b16841578af4e500528264c9] into tmpdir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp, totalSize=28.2 K 2024-11-10T20:27:41,467 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ef522d6670c49e180aee6a40e50c26c, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731270454965 2024-11-10T20:27:41,468 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1641821d99914102b85d3d62bba5cb40, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731270458990 2024-11-10T20:27:41,468 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 657fecc6b16841578af4e500528264c9, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731270461031 2024-11-10T20:27:41,481 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d422e1883575bc1a2891d04cf7083b0b#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:27:41,481 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/b080dc269945422b8ed9830b72f4333f is 1080, key is row0002/info:/1731270454965/Put/seqid=0 2024-11-10T20:27:41,483 WARN [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,483 WARN [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK], DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]) is bad. 2024-11-10T20:27:41,483 WARN [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741869_1052 2024-11-10T20:27:41,484 WARN [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK] 2024-11-10T20:27:41,485 WARN [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,485 WARN [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK], DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]) is bad. 2024-11-10T20:27:41,485 WARN [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741870_1053 2024-11-10T20:27:41,486 WARN [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK] 2024-11-10T20:27:41,487 WARN [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,487 WARN [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:41,487 WARN [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741871_1054 2024-11-10T20:27:41,487 WARN [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:41,489 WARN [Thread-950 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33771 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:41,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34448 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741872_1055] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741872_1055 to mirror 127.0.0.1:33771 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:41,489 WARN [Thread-950 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:41,489 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34448 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741872_1055] {}] datanode.BlockReceiver(316): Block 1073741872 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:41,489 WARN [Thread-950 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741872_1055 2024-11-10T20:27:41,490 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34448 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741872_1055] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34448 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:41,490 WARN [Thread-950 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] 2024-11-10T20:27:41,491 WARN [IPC Server handler 0 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T20:27:41,491 WARN [IPC Server handler 0 on default port 34715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T20:27:41,491 WARN [IPC Server handler 0 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T20:27:41,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741873_1056 (size=17994) 2024-11-10T20:27:41,901 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/b080dc269945422b8ed9830b72f4333f as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f 2024-11-10T20:27:41,907 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d422e1883575bc1a2891d04cf7083b0b/info of d422e1883575bc1a2891d04cf7083b0b into b080dc269945422b8ed9830b72f4333f(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d422e1883575bc1a2891d04cf7083b0b: 2024-11-10T20:27:41,908 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b., storeName=d422e1883575bc1a2891d04cf7083b0b/info, priority=13, startTime=1731270461465; duration=0sec 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f because midkey is the same as first or last row 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f because midkey is the same as first or last row 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f because midkey is the same as first or last row 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:27:41,908 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d422e1883575bc1a2891d04cf7083b0b:info 2024-11-10T20:27:42,283 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45329 {}] regionserver.HRegion(8855): Flush requested on d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:42,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d422e1883575bc1a2891d04cf7083b0b 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T20:27:42,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/bf986c256f1f40319b93e8ae9cf9ef8d is 1079, key is tmprow/info:/1731270462449/Put/seqid=0 2024-11-10T20:27:42,457 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:42,457 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]) is bad. 2024-11-10T20:27:42,457 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741874_1057 2024-11-10T20:27:42,458 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK] 2024-11-10T20:27:42,460 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33771 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:42,460 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34456 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741875_1058 to mirror 127.0.0.1:33771 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:42,460 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]) is bad. 2024-11-10T20:27:42,460 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741875_1058 2024-11-10T20:27:42,460 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34456 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:42,460 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34456 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34456 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:42,461 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33771,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK] 2024-11-10T20:27:42,463 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34905 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:42,463 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34458 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741876_1059 to mirror 127.0.0.1:34905 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:42,463 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK]) is bad. 2024-11-10T20:27:42,463 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34458 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:42,463 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741876_1059 2024-11-10T20:27:42,463 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34458 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34458 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:42,463 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34905,DS-ab0f1e07-554d-4c96-804e-8417d9c98653,DISK] 2024-11-10T20:27:42,466 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37785 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:42,465 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34468 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741877_1060 to mirror 127.0.0.1:37785 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:42,466 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:42,466 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741877_1060 2024-11-10T20:27:42,466 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34468 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:42,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:34468 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34468 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:42,466 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:42,467 WARN [IPC Server handler 4 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T20:27:42,467 WARN [IPC Server handler 4 on default port 34715 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T20:27:42,467 WARN [IPC Server handler 4 on default port 34715 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T20:27:42,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741878_1061 (size=6027) 2024-11-10T20:27:42,751 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@35571ba4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741843_1026 to 127.0.0.1:34905 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:42,751 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6896059c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741853_1036 to 127.0.0.1:34905 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:42,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/bf986c256f1f40319b93e8ae9cf9ef8d 2024-11-10T20:27:42,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/bf986c256f1f40319b93e8ae9cf9ef8d as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/bf986c256f1f40319b93e8ae9cf9ef8d 2024-11-10T20:27:42,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/bf986c256f1f40319b93e8ae9cf9ef8d, entries=1, sequenceid=45, filesize=5.9 K 2024-11-10T20:27:42,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for d422e1883575bc1a2891d04cf7083b0b in 434ms, sequenceid=45, compaction requested=false 2024-11-10T20:27:42,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d422e1883575bc1a2891d04cf7083b0b: 2024-11-10T20:27:42,885 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K 2024-11-10T20:27:42,885 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:42,885 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f because midkey is the same as first or last row 2024-11-10T20:27:43,011 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:43,012 WARN [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-11-10T20:27:43,065 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:27:43,068 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:27:43,069 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:27:43,069 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:27:43,069 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:27:43,069 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@66e63d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:27:43,070 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d04364e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:27:43,184 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@633469fc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/java.io.tmpdir/jetty-localhost-45719-hadoop-hdfs-3_4_1-tests_jar-_-any-17504109095943118781/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:27:43,185 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@28f4e296{HTTP/1.1, (http/1.1)}{localhost:45719} 2024-11-10T20:27:43,185 INFO [Time-limited test {}] server.Server(415): Started @125659ms 2024-11-10T20:27:43,186 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:27:43,281 WARN [Thread-979 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:27:43,289 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbcd42c9dc325ba91 with lease ID 0xa6dd06c378e0d829: from storage DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe node DatanodeRegistration(127.0.0.1:39639, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=45277, infoSecurePort=0, ipcPort=36985, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T20:27:43,289 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbcd42c9dc325ba91 with lease ID 0xa6dd06c378e0d829: from storage DS-f3451470-18c7-466b-a8f6-02b842974553 node DatanodeRegistration(127.0.0.1:39639, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=45277, infoSecurePort=0, ipcPort=36985, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:27:43,340 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:43,751 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6896059c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741858_1041 to 127.0.0.1:34905 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:43,751 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@35571ba4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741848_1031 to 127.0.0.1:42605 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:44,284 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:45,012 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:45,340 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:45,752 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@35571ba4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741868_1051 to 127.0.0.1:37785 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:45,752 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6896059c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741873_1056 to 127.0.0.1:37785 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:46,284 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:46,751 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@35571ba4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46141, datanodeUuid=63e99a1c-a869-4ef8-9197-ea53d071eae7, infoPort=39841, infoSecurePort=0, ipcPort=46173, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741878_1061 to 127.0.0.1:34905 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:47,012 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:47,341 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:48,285 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:49,013 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:49,341 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:50,285 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:51,013 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:51,200 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T20:27:51,341 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:51,434 ERROR [FSHLog-0-hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData-prefix:734d44df782b,40057,1731270441219 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:51,435 WARN [FSHLog-0-hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData-prefix:734d44df782b,40057,1731270441219 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:51,435 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C40057%2C1731270441219:(num 1731270441358) roll requested 2024-11-10T20:27:51,435 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C40057%2C1731270441219.1731270471435 2024-11-10T20:27:51,438 WARN [Thread-999 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:51,438 WARN [Thread-999 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK], DatanodeInfoWithStorage[127.0.0.1:39639,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:51,438 WARN [Thread-999 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741879_1062 2024-11-10T20:27:51,439 WARN [Thread-999 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:51,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:51,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:51,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:51,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:51,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:51,444 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270441358 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270471435 2024-11-10T20:27:51,444 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:51,444 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:51,445 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270441358 2024-11-10T20:27:51,445 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45277:45277),(127.0.0.1/127.0.0.1:39841:39841)] 2024-11-10T20:27:51,445 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270441358 is not closed yet, will try archiving it next time 2024-11-10T20:27:51,445 WARN [IPC Server handler 2 on default port 34715 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270441358 has not been closed. Lease recovery is in progress. RecoveryId = 1064 for block blk_1073741830_1006 2024-11-10T20:27:51,445 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270441358 after 0ms 2024-11-10T20:27:52,285 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:53,013 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:53,305 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@10b2f26e {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1984011214-172.17.0.2-1731270440526:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:42605,null,null]) java.net.ConnectException: Call From 734d44df782b/172.17.0.2 to localhost:38207 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-10T20:27:53,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741833_1020 (size=455) 2024-11-10T20:27:53,980 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270441661 to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs/734d44df782b%2C45329%2C1731270441266.1731270441661 2024-11-10T20:27:53,982 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270458974 to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs/734d44df782b%2C45329%2C1731270441266.1731270458974 2024-11-10T20:27:54,286 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:54,287 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@21e8273d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39639, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=45277, infoSecurePort=0, ipcPort=36985, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741833_1020 to 127.0.0.1:42605 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:55,014 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:55,446 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/WALs/734d44df782b,40057,1731270441219/734d44df782b%2C40057%2C1731270441219.1731270441358 after 4001ms 2024-11-10T20:27:56,286 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:57,014 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:57,287 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@21e8273d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39639, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=45277, infoSecurePort=0, ipcPort=36985, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741835_1011 to 127.0.0.1:37785 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:57,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:27:58,287 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:58,287 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@773ba02a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39639, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=45277, infoSecurePort=0, ipcPort=36985, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741829_1005 to 127.0.0.1:37785 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:58,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:27:58,925 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C45329%2C1731270441266.1731270478925 2024-11-10T20:27:58,928 WARN [Thread-1012 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1065 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:58,928 WARN [Thread-1012 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741881_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK], DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:58,928 WARN [Thread-1012 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741881_1065 2024-11-10T20:27:58,929 WARN [Thread-1012 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:58,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:58,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:58,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:58,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:58,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:58,934 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270460997 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270478925 2024-11-10T20:27:58,934 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39841:39841),(127.0.0.1/127.0.0.1:45277:45277)] 2024-11-10T20:27:58,935 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270460997 is not closed yet, will try archiving it next time 2024-11-10T20:27:58,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741863_1046 (size=13591) 2024-11-10T20:27:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45329 {}] regionserver.HRegion(8855): Flush requested on d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:58,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d422e1883575bc1a2891d04cf7083b0b 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T20:27:58,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/3d7170fd00aa45898839a9b5fd75d056 is 1080, key is row0013/info:/1731270478936/Put/seqid=0 2024-11-10T20:27:58,951 WARN [Thread-1018 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1067 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:58,951 WARN [Thread-1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741883_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK], DatanodeInfoWithStorage[127.0.0.1:39639,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:58,951 WARN [Thread-1018 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741883_1067 2024-11-10T20:27:58,952 WARN [Thread-1018 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:58,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741884_1068 (size=11421) 2024-11-10T20:27:58,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741884_1068 (size=11421) 2024-11-10T20:27:58,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/3d7170fd00aa45898839a9b5fd75d056 2024-11-10T20:27:58,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/3d7170fd00aa45898839a9b5fd75d056 as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3d7170fd00aa45898839a9b5fd75d056 2024-11-10T20:27:58,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3d7170fd00aa45898839a9b5fd75d056, entries=6, sequenceid=55, filesize=11.2 K 2024-11-10T20:27:58,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7530, heapSize ~8.11 KB/8304, currentSize=6.30 KB/6455 for d422e1883575bc1a2891d04cf7083b0b in 28ms, sequenceid=55, compaction requested=true 2024-11-10T20:27:58,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d422e1883575bc1a2891d04cf7083b0b: 2024-11-10T20:27:58,972 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=34.6 K, sizeToCheck=16.0 K 2024-11-10T20:27:58,972 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:58,972 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f because midkey is the same as first or last row 2024-11-10T20:27:58,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d422e1883575bc1a2891d04cf7083b0b:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:27:58,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:27:58,972 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:27:58,974 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:27:58,974 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HStore(1541): d422e1883575bc1a2891d04cf7083b0b/info is initiating minor compaction (all files) 2024-11-10T20:27:58,974 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d422e1883575bc1a2891d04cf7083b0b/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:58,974 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/bf986c256f1f40319b93e8ae9cf9ef8d, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3d7170fd00aa45898839a9b5fd75d056] into tmpdir=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp, totalSize=34.6 K 2024-11-10T20:27:58,974 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.Compactor(225): Compacting b080dc269945422b8ed9830b72f4333f, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731270454965 2024-11-10T20:27:58,975 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf986c256f1f40319b93e8ae9cf9ef8d, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731270462449 2024-11-10T20:27:58,975 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d7170fd00aa45898839a9b5fd75d056, keycount=6, bloomtype=ROW, size=11.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731270462854 2024-11-10T20:27:58,992 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d422e1883575bc1a2891d04cf7083b0b#info#compaction#24 average throughput is 8.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:27:58,993 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/3f6c891dd6f64f00bd9be471d68ef29d is 1080, key is row0002/info:/1731270454965/Put/seqid=0 2024-11-10T20:27:58,996 WARN [Thread-1027 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1069 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37785 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:58,995 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:43456 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741885_1069] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741885_1069 to mirror 127.0.0.1:37785 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:58,996 WARN [Thread-1027 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741885_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:58,996 WARN [Thread-1027 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741885_1069 2024-11-10T20:27:58,996 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:43456 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741885_1069] {}] datanode.BlockReceiver(316): Block 1073741885 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:58,996 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:43456 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741885_1069] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43456 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:58,996 WARN [Thread-1027 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:59,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741886_1070 (size=23502) 2024-11-10T20:27:59,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741886_1070 (size=23502) 2024-11-10T20:27:59,007 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/3f6c891dd6f64f00bd9be471d68ef29d as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3f6c891dd6f64f00bd9be471d68ef29d 2024-11-10T20:27:59,014 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d422e1883575bc1a2891d04cf7083b0b/info of d422e1883575bc1a2891d04cf7083b0b into 3f6c891dd6f64f00bd9be471d68ef29d(size=23.0 K), total size for store is 23.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:27:59,014 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d422e1883575bc1a2891d04cf7083b0b: 2024-11-10T20:27:59,014 INFO [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b., storeName=d422e1883575bc1a2891d04cf7083b0b/info, priority=13, startTime=1731270478972; duration=0sec 2024-11-10T20:27:59,014 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,015 INFO [regionserver/734d44df782b:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled. 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3f6c891dd6f64f00bd9be471d68ef29d because midkey is the same as first or last row 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3f6c891dd6f64f00bd9be471d68ef29d because midkey is the same as first or last row 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3f6c891dd6f64f00bd9be471d68ef29d because midkey is the same as first or last row 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:27:59,015 DEBUG [RS:0;734d44df782b:45329-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d422e1883575bc1a2891d04cf7083b0b:info 2024-11-10T20:27:59,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T20:27:59,158 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:27:59,158 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:59,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:59,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:59,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T20:27:59,158 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T20:27:59,158 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=879908278, stopped=false 2024-11-10T20:27:59,159 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=734d44df782b,40057,1731270441219 2024-11-10T20:27:59,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:59,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:59,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:59,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:59,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:27:59,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:27:59,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:27:59,161 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:27:59,161 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:59,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:59,161 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '734d44df782b,45329,1731270441266' ***** 2024-11-10T20:27:59,161 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T20:27:59,161 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '734d44df782b,43225,1731270442226' ***** 2024-11-10T20:27:59,161 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T20:27:59,162 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:59,162 INFO [RS:1;734d44df782b:43225 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T20:27:59,162 INFO [RS:0;734d44df782b:45329 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T20:27:59,162 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:59,162 INFO [RS:1;734d44df782b:43225 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T20:27:59,162 INFO [RS:0;734d44df782b:45329 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T20:27:59,162 INFO [RS:1;734d44df782b:43225 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T20:27:59,162 INFO [RS:0;734d44df782b:45329 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T20:27:59,162 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T20:27:59,162 INFO [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(959): stopping server 734d44df782b,43225,1731270442226 2024-11-10T20:27:59,162 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:27:59,162 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(3091): Received CLOSE for d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:59,162 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T20:27:59,163 INFO [RS:1;734d44df782b:43225 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:27:59,163 INFO [RS:1;734d44df782b:43225 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;734d44df782b:43225. 2024-11-10T20:27:59,163 DEBUG [RS:1;734d44df782b:43225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:59,163 DEBUG [RS:1;734d44df782b:43225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:59,163 INFO [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(976): stopping server 734d44df782b,43225,1731270442226; all regions closed. 2024-11-10T20:27:59,164 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(959): stopping server 734d44df782b,45329,1731270441266 2024-11-10T20:27:59,164 INFO [RS:0;734d44df782b:45329 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:27:59,164 INFO [RS:0;734d44df782b:45329 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;734d44df782b:45329. 2024-11-10T20:27:59,164 DEBUG [RS:0;734d44df782b:45329 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:27:59,164 DEBUG [RS:0;734d44df782b:45329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:27:59,164 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d422e1883575bc1a2891d04cf7083b0b, disabling compactions & flushes 2024-11-10T20:27:59,164 INFO [RS:0;734d44df782b:45329 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T20:27:59,164 INFO [RS:0;734d44df782b:45329 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T20:27:59,164 INFO [RS:0;734d44df782b:45329 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T20:27:59,164 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:59,164 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T20:27:59,164 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:59,164 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. after waiting 0 ms 2024-11-10T20:27:59,164 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:59,164 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T20:27:59,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,164 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d422e1883575bc1a2891d04cf7083b0b=TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.} 2024-11-10T20:27:59,164 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d422e1883575bc1a2891d04cf7083b0b 1/1 column families, dataSize=6.30 KB heapSize=7 KB 2024-11-10T20:27:59,164 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d422e1883575bc1a2891d04cf7083b0b 2024-11-10T20:27:59,164 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:27:59,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,164 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:27:59,164 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:27:59,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,165 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:27:59,165 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:27:59,165 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-11-10T20:27:59,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,165 ERROR [FSHLog-0-hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad-prefix:734d44df782b,45329,1731270441266.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,165 WARN [FSHLog-0-hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad-prefix:734d44df782b,45329,1731270441266.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,165 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C45329%2C1731270441266.meta:.meta(num 1731270442046) roll requested 2024-11-10T20:27:59,166 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C45329%2C1731270441266.meta.1731270479166.meta 2024-11-10T20:27:59,166 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,166 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,166 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 2024-11-10T20:27:59,167 WARN [IPC Server handler 3 on default port 34715 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 has not been closed. Lease recovery is in progress. RecoveryId = 1071 for block blk_1073741837_1013 2024-11-10T20:27:59,167 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 after 1ms 2024-11-10T20:27:59,170 WARN [Thread-1034 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1072 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37785 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,170 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:40154 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741887_1072] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data4]'}, localName='127.0.0.1:39639', datanodeUuid='422d81f1-121b-44ef-a243-9c418ef1c7dc', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741887_1072 to mirror 127.0.0.1:37785 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:59,170 WARN [Thread-1034 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741887_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39639,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:59,170 WARN [Thread-1034 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741887_1072 2024-11-10T20:27:59,171 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:40154 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741887_1072] {}] datanode.BlockReceiver(316): Block 1073741887 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T20:27:59,171 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:40154 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741887_1072] {}] datanode.DataXceiver(331): 127.0.0.1:39639:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40154 dst: /127.0.0.1:39639 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:59,171 WARN [Thread-1034 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:59,171 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/cc59dbac3f534e50b4a8ef082feb1ca7 is 1080, key is row0018/info:/1731270478945/Put/seqid=0 2024-11-10T20:27:59,174 WARN [Thread-1035 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1074 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37785 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,174 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:43476 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741889_1074] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8]'}, localName='127.0.0.1:46141', datanodeUuid='63e99a1c-a869-4ef8-9197-ea53d071eae7', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741889_1074 to mirror 127.0.0.1:37785 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:59,174 WARN [Thread-1035 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741889_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46141,DS-8ba33af9-ad3d-41ca-ba39-0c02f073bdb8,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:59,175 WARN [Thread-1035 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741889_1074 2024-11-10T20:27:59,175 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:43476 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741889_1074] {}] datanode.BlockReceiver(316): Block 1073741889 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:59,175 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:43476 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741889_1074] {}] datanode.DataXceiver(331): 127.0.0.1:46141:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43476 dst: /127.0.0.1:46141 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:59,175 WARN [Thread-1035 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:59,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,176 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270479166.meta 2024-11-10T20:27:59,177 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,177 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42605,DS-185c4c90-f6eb-4ed2-8827-fe4f4014c87b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,177 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta 2024-11-10T20:27:59,177 WARN [IPC Server handler 3 on default port 34715 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta has not been closed. Lease recovery is in progress. RecoveryId = 1076 for block blk_1073741834_1010 2024-11-10T20:27:59,178 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta after 1ms 2024-11-10T20:27:59,181 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39841:39841),(127.0.0.1/127.0.0.1:45277:45277)] 2024-11-10T20:27:59,181 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta is not closed yet, will try archiving it next time 2024-11-10T20:27:59,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741890_1075 (size=11421) 2024-11-10T20:27:59,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741890_1075 (size=11421) 2024-11-10T20:27:59,184 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.30 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/cc59dbac3f534e50b4a8ef082feb1ca7 2024-11-10T20:27:59,191 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/.tmp/info/cc59dbac3f534e50b4a8ef082feb1ca7 as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/cc59dbac3f534e50b4a8ef082feb1ca7 2024-11-10T20:27:59,198 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/cc59dbac3f534e50b4a8ef082feb1ca7, entries=6, sequenceid=65, filesize=11.2 K 2024-11-10T20:27:59,200 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~6.30 KB/6455, heapSize ~6.98 KB/7152, currentSize=0 B/0 for d422e1883575bc1a2891d04cf7083b0b in 35ms, sequenceid=65, compaction requested=false 2024-11-10T20:27:59,200 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/6ef522d6670c49e180aee6a40e50c26c, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/1641821d99914102b85d3d62bba5cb40, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/657fecc6b16841578af4e500528264c9, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/bf986c256f1f40319b93e8ae9cf9ef8d, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3d7170fd00aa45898839a9b5fd75d056] to archive 2024-11-10T20:27:59,201 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T20:27:59,203 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/.tmp/info/35513ffa1e0c465ead4cb3c215854e5b is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b./info:regioninfo/1731270442675/Put/seqid=0 2024-11-10T20:27:59,203 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/6ef522d6670c49e180aee6a40e50c26c to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/6ef522d6670c49e180aee6a40e50c26c 2024-11-10T20:27:59,205 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/1641821d99914102b85d3d62bba5cb40 to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/1641821d99914102b85d3d62bba5cb40 2024-11-10T20:27:59,207 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/b080dc269945422b8ed9830b72f4333f 2024-11-10T20:27:59,209 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/657fecc6b16841578af4e500528264c9 to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/657fecc6b16841578af4e500528264c9 2024-11-10T20:27:59,211 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/bf986c256f1f40319b93e8ae9cf9ef8d to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/bf986c256f1f40319b93e8ae9cf9ef8d 2024-11-10T20:27:59,212 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3d7170fd00aa45898839a9b5fd75d056 to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/info/3d7170fd00aa45898839a9b5fd75d056 2024-11-10T20:27:59,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741891_1077 (size=7089) 2024-11-10T20:27:59,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741891_1077 (size=7089) 2024-11-10T20:27:59,212 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=734d44df782b:40057 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-10T20:27:59,213 WARN [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [6ef522d6670c49e180aee6a40e50c26c=10347, 1641821d99914102b85d3d62bba5cb40=12506, b080dc269945422b8ed9830b72f4333f=17994, 657fecc6b16841578af4e500528264c9=6027, bf986c256f1f40319b93e8ae9cf9ef8d=6027, 3d7170fd00aa45898839a9b5fd75d056=11421] 2024-11-10T20:27:59,217 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/d422e1883575bc1a2891d04cf7083b0b/recovered.edits/68.seqid, newMaxSeqId=68, maxSeqId=1 2024-11-10T20:27:59,217 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:59,218 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d422e1883575bc1a2891d04cf7083b0b: Waiting for close lock at 1731270479164Running coprocessor pre-close hooks at 1731270479164Disabling compacts and flushes for region at 1731270479164Disabling writes for close at 1731270479164Obtaining lock to block concurrent updates at 1731270479164Preparing flush snapshotting stores in d422e1883575bc1a2891d04cf7083b0b at 1731270479164Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b., syncing WAL and waiting on mvcc, flushsize=dataSize=6455, getHeapSize=7152, getOffHeapSize=0, getCellsCount=6 at 1731270479165 (+1 ms)Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. at 1731270479166 (+1 ms)Flushing d422e1883575bc1a2891d04cf7083b0b/info: creating writer at 1731270479166Flushing d422e1883575bc1a2891d04cf7083b0b/info: appending metadata at 1731270479171 (+5 ms)Flushing d422e1883575bc1a2891d04cf7083b0b/info: closing flushed file at 1731270479171Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4339cf71: reopening flushed file at 1731270479190 (+19 ms)Finished flush of dataSize ~6.30 KB/6455, heapSize ~6.98 KB/7152, currentSize=0 B/0 for d422e1883575bc1a2891d04cf7083b0b in 35ms, sequenceid=65, compaction requested=false at 1731270479200 (+10 ms)Writing region close event to WAL at 1731270479213 (+13 ms)Running coprocessor post-close hooks at 1731270479217 (+4 ms)Closed at 1731270479217 2024-11-10T20:27:59,218 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731270442315.d422e1883575bc1a2891d04cf7083b0b. 2024-11-10T20:27:59,336 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.1731270460997 to hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs/734d44df782b%2C45329%2C1731270441266.1731270460997 2024-11-10T20:27:59,340 INFO [regionserver/734d44df782b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T20:27:59,340 INFO [regionserver/734d44df782b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T20:27:59,365 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T20:27:59,520 INFO [regionserver/734d44df782b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T20:27:59,520 INFO [regionserver/734d44df782b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T20:27:59,522 INFO [regionserver/734d44df782b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:27:59,565 DEBUG [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T20:27:59,612 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/.tmp/info/35513ffa1e0c465ead4cb3c215854e5b 2024-11-10T20:27:59,634 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/.tmp/ns/6459ed4e732548e384fe0267b11d0718 is 43, key is default/ns:d/1731270442089/Put/seqid=0 2024-11-10T20:27:59,637 WARN [Thread-1056 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1078 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37785 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,637 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:40178 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741892_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data4]'}, localName='127.0.0.1:39639', datanodeUuid='422d81f1-121b-44ef-a243-9c418ef1c7dc', xmitsInProgress=0}:Exception transferring block BP-1984011214-172.17.0.2-1731270440526:blk_1073741892_1078 to mirror 127.0.0.1:37785 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:59,637 WARN [Thread-1056 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741892_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39639,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK], DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:59,637 WARN [Thread-1056 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741892_1078 2024-11-10T20:27:59,637 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:40178 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741892_1078] {}] datanode.BlockReceiver(316): Block 1073741892 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T20:27:59,638 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1896572732_22 at /127.0.0.1:40178 [Receiving block BP-1984011214-172.17.0.2-1731270440526:blk_1073741892_1078] {}] datanode.DataXceiver(331): 127.0.0.1:39639:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40178 dst: /127.0.0.1:39639 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:27:59,638 WARN [Thread-1056 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:59,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741893_1079 (size=5153) 2024-11-10T20:27:59,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741893_1079 (size=5153) 2024-11-10T20:27:59,644 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/.tmp/ns/6459ed4e732548e384fe0267b11d0718 2024-11-10T20:27:59,666 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/.tmp/table/ee4f91c978564e2cab8990dbdc64addb is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731270442686/Put/seqid=0 2024-11-10T20:27:59,668 WARN [Thread-1063 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1080 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:27:59,668 WARN [Thread-1063 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1984011214-172.17.0.2-1731270440526:blk_1073741894_1080 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK], DatanodeInfoWithStorage[127.0.0.1:39639,DS-6eea2ddd-40d2-4b9c-8424-62a6df522bbe,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK]) is bad. 2024-11-10T20:27:59,668 WARN [Thread-1063 {}] hdfs.DataStreamer(1850): Abandoning BP-1984011214-172.17.0.2-1731270440526:blk_1073741894_1080 2024-11-10T20:27:59,669 WARN [Thread-1063 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37785,DS-fd121265-d2bb-4b39-9dac-db7030c4a90c,DISK] 2024-11-10T20:27:59,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741895_1081 (size=5424) 2024-11-10T20:27:59,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741895_1081 (size=5424) 2024-11-10T20:27:59,674 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/.tmp/table/ee4f91c978564e2cab8990dbdc64addb 2024-11-10T20:27:59,681 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/.tmp/info/35513ffa1e0c465ead4cb3c215854e5b as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/info/35513ffa1e0c465ead4cb3c215854e5b 2024-11-10T20:27:59,686 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/info/35513ffa1e0c465ead4cb3c215854e5b, entries=10, sequenceid=11, filesize=6.9 K 2024-11-10T20:27:59,687 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/.tmp/ns/6459ed4e732548e384fe0267b11d0718 as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/ns/6459ed4e732548e384fe0267b11d0718 2024-11-10T20:27:59,693 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/ns/6459ed4e732548e384fe0267b11d0718, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T20:27:59,694 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/.tmp/table/ee4f91c978564e2cab8990dbdc64addb as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/table/ee4f91c978564e2cab8990dbdc64addb 2024-11-10T20:27:59,699 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/table/ee4f91c978564e2cab8990dbdc64addb, entries=2, sequenceid=11, filesize=5.3 K 2024-11-10T20:27:59,701 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 535ms, sequenceid=11, compaction requested=false 2024-11-10T20:27:59,707 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T20:27:59,708 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:27:59,708 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:27:59,708 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270479164Running coprocessor pre-close hooks at 1731270479164Disabling compacts and flushes for region at 1731270479164Disabling writes for close at 1731270479165 (+1 ms)Obtaining lock to block concurrent updates at 1731270479165Preparing flush snapshotting stores in 1588230740 at 1731270479165Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731270479165Flushing stores of hbase:meta,,1.1588230740 at 1731270479181 (+16 ms)Flushing 1588230740/info: creating writer at 1731270479181Flushing 1588230740/info: appending metadata at 1731270479202 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731270479202Flushing 1588230740/ns: creating writer at 1731270479618 (+416 ms)Flushing 1588230740/ns: appending metadata at 1731270479634 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731270479634Flushing 1588230740/table: creating writer at 1731270479650 (+16 ms)Flushing 1588230740/table: appending metadata at 1731270479665 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731270479665Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@502062f6: reopening flushed file at 1731270479680 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65629db0: reopening flushed file at 1731270479687 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39ae0ad3: reopening flushed file at 1731270479693 (+6 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 535ms, sequenceid=11, compaction requested=false at 1731270479701 (+8 ms)Writing region close event to WAL at 1731270479704 (+3 ms)Running coprocessor post-close hooks at 1731270479708 (+4 ms)Closed at 1731270479708 2024-11-10T20:27:59,708 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T20:27:59,765 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(976): stopping server 734d44df782b,45329,1731270441266; all regions closed. 2024-11-10T20:27:59,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:27:59,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741888_1073 (size=825) 2024-11-10T20:27:59,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741888_1073 (size=825) 2024-11-10T20:28:00,283 INFO [regionserver/734d44df782b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:28:00,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:28:00,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:28:00,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741863_1046 (size=13591) 2024-11-10T20:28:01,288 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@773ba02a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39639, datanodeUuid=422d81f1-121b-44ef-a243-9c418ef1c7dc, infoPort=45277, infoSecurePort=0, ipcPort=36985, storageInfo=lv=-57;cid=testClusterID;nsid=805609783;c=1731270440526):Failed to transfer BP-1984011214-172.17.0.2-1731270440526:blk_1073741826_1002 to 127.0.0.1:37785 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:02,105 INFO [master/734d44df782b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T20:28:02,105 INFO [master/734d44df782b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T20:28:02,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T20:28:02,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:28:02,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T20:28:03,168 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 after 4002ms 2024-11-10T20:28:03,178 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta after 4001ms 2024-11-10T20:28:03,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:28:03,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741836_1012 (size=76) 2024-11-10T20:28:03,309 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@f8b2bc9 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1984011214-172.17.0.2-1731270440526:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:42605,null,null]) java.net.ConnectException: Call From 734d44df782b/172.17.0.2 to localhost:38207 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-10T20:28:04,166 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-10T20:28:04,169 DEBUG [RS:1;734d44df782b:43225 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs 2024-11-10T20:28:04,169 INFO [RS:1;734d44df782b:43225 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C43225%2C1731270442226:(num 1731270442416) 2024-11-10T20:28:04,169 DEBUG [RS:1;734d44df782b:43225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:28:04,169 INFO [RS:1;734d44df782b:43225 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:28:04,169 INFO [RS:1;734d44df782b:43225 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:28:04,169 INFO [RS:1;734d44df782b:43225 {}] hbase.ChoreService(370): Chore service for: regionserver/734d44df782b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T20:28:04,169 INFO [RS:1;734d44df782b:43225 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T20:28:04,169 INFO [RS:1;734d44df782b:43225 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T20:28:04,169 INFO [RS:1;734d44df782b:43225 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T20:28:04,169 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:28:04,169 INFO [RS:1;734d44df782b:43225 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:28:04,170 INFO [RS:1;734d44df782b:43225 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43225 2024-11-10T20:28:04,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/734d44df782b,43225,1731270442226 2024-11-10T20:28:04,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:28:04,172 INFO [RS:1;734d44df782b:43225 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:28:04,174 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [734d44df782b,43225,1731270442226] 2024-11-10T20:28:04,174 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:04,175 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/734d44df782b,43225,1731270442226 already deleted, retry=false 2024-11-10T20:28:04,175 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 734d44df782b,43225,1731270442226 expired; onlineServers=1 2024-11-10T20:28:04,218 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,234 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,234 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,235 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,235 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,235 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,242 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,243 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:04,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43225-0x10193ddee5a0002, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:04,274 INFO [RS:1;734d44df782b:43225 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:28:04,274 INFO [RS:1;734d44df782b:43225 {}] regionserver.HRegionServer(1031): Exiting; stopping=734d44df782b,43225,1731270442226; zookeeper connection closed. 2024-11-10T20:28:04,274 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@35dc2118 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@35dc2118 2024-11-10T20:28:04,745 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T20:28:04,760 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,761 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,761 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,762 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,762 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,762 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,766 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,766 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-10T20:28:04,769 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:04,770 DEBUG [RS:0;734d44df782b:45329 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs 2024-11-10T20:28:04,770 INFO [RS:0;734d44df782b:45329 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C45329%2C1731270441266.meta:.meta(num 1731270479166) 2024-11-10T20:28:04,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741882_1066 (size=15140) 2024-11-10T20:28:04,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741882_1066 (size=15140) 2024-11-10T20:28:04,776 DEBUG [RS:0;734d44df782b:45329 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/oldWALs 2024-11-10T20:28:04,776 INFO [RS:0;734d44df782b:45329 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C45329%2C1731270441266:(num 1731270478925) 2024-11-10T20:28:04,776 DEBUG [RS:0;734d44df782b:45329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:28:04,776 INFO [RS:0;734d44df782b:45329 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:28:04,776 INFO [RS:0;734d44df782b:45329 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:28:04,776 INFO [RS:0;734d44df782b:45329 {}] hbase.ChoreService(370): Chore service for: regionserver/734d44df782b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T20:28:04,776 INFO [RS:0;734d44df782b:45329 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:28:04,776 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:28:04,776 INFO [RS:0;734d44df782b:45329 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45329 2024-11-10T20:28:04,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/734d44df782b,45329,1731270441266 2024-11-10T20:28:04,778 INFO [RS:0;734d44df782b:45329 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:28:04,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:28:04,780 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [734d44df782b,45329,1731270441266] 2024-11-10T20:28:04,782 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/734d44df782b,45329,1731270441266 already deleted, retry=false 2024-11-10T20:28:04,782 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 734d44df782b,45329,1731270441266 expired; onlineServers=0 2024-11-10T20:28:04,782 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '734d44df782b,40057,1731270441219' ***** 2024-11-10T20:28:04,782 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T20:28:04,782 INFO [M:0;734d44df782b:40057 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:28:04,782 INFO [M:0;734d44df782b:40057 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:28:04,782 DEBUG [M:0;734d44df782b:40057 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T20:28:04,782 DEBUG [M:0;734d44df782b:40057 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T20:28:04,782 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T20:28:04,782 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270441435 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270441435,5,FailOnTimeoutGroup] 2024-11-10T20:28:04,782 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270441434 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270441434,5,FailOnTimeoutGroup] 2024-11-10T20:28:04,783 INFO [M:0;734d44df782b:40057 {}] hbase.ChoreService(370): Chore service for: master/734d44df782b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T20:28:04,783 INFO [M:0;734d44df782b:40057 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:28:04,783 DEBUG [M:0;734d44df782b:40057 {}] master.HMaster(1795): Stopping service threads 2024-11-10T20:28:04,783 INFO [M:0;734d44df782b:40057 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T20:28:04,783 INFO [M:0;734d44df782b:40057 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:28:04,783 INFO [M:0;734d44df782b:40057 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T20:28:04,783 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T20:28:04,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T20:28:04,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:04,785 DEBUG [M:0;734d44df782b:40057 {}] zookeeper.ZKUtil(347): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T20:28:04,785 WARN [M:0;734d44df782b:40057 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T20:28:04,785 INFO [M:0;734d44df782b:40057 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/.lastflushedseqids 2024-11-10T20:28:04,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741896_1082 (size=130) 2024-11-10T20:28:04,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741896_1082 (size=130) 2024-11-10T20:28:04,792 INFO [M:0;734d44df782b:40057 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T20:28:04,793 INFO [M:0;734d44df782b:40057 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T20:28:04,793 DEBUG [M:0;734d44df782b:40057 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:28:04,793 INFO [M:0;734d44df782b:40057 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:04,793 DEBUG [M:0;734d44df782b:40057 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:04,793 DEBUG [M:0;734d44df782b:40057 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:28:04,793 DEBUG [M:0;734d44df782b:40057 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:04,793 INFO [M:0;734d44df782b:40057 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB 2024-11-10T20:28:04,810 DEBUG [M:0;734d44df782b:40057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/86224bebedb24b6cb011c98909ffc615 is 82, key is hbase:meta,,1/info:regioninfo/1731270442072/Put/seqid=0 2024-11-10T20:28:04,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741897_1083 (size=5672) 2024-11-10T20:28:04,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741897_1083 (size=5672) 2024-11-10T20:28:04,816 INFO [M:0;734d44df782b:40057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/86224bebedb24b6cb011c98909ffc615 2024-11-10T20:28:04,841 DEBUG [M:0;734d44df782b:40057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb0dabc789334128bdbc056021d8e375 is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731270442692/Put/seqid=0 2024-11-10T20:28:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741898_1084 (size=6256) 2024-11-10T20:28:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741898_1084 (size=6256) 2024-11-10T20:28:04,847 INFO [M:0;734d44df782b:40057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb0dabc789334128bdbc056021d8e375 2024-11-10T20:28:04,852 INFO [M:0;734d44df782b:40057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb0dabc789334128bdbc056021d8e375 2024-11-10T20:28:04,867 DEBUG [M:0;734d44df782b:40057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb2bef8f1b6846b3a69e9e2f3be69c39 is 69, key is 734d44df782b,43225,1731270442226/rs:state/1731270442269/Put/seqid=0 2024-11-10T20:28:04,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741899_1085 (size=5224) 2024-11-10T20:28:04,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741899_1085 (size=5224) 2024-11-10T20:28:04,873 INFO [M:0;734d44df782b:40057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb2bef8f1b6846b3a69e9e2f3be69c39 2024-11-10T20:28:04,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:04,880 INFO [RS:0;734d44df782b:45329 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:28:04,880 INFO [RS:0;734d44df782b:45329 {}] regionserver.HRegionServer(1031): Exiting; stopping=734d44df782b,45329,1731270441266; zookeeper connection closed. 2024-11-10T20:28:04,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45329-0x10193ddee5a0001, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:04,881 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c81ac21 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c81ac21 2024-11-10T20:28:04,881 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-10T20:28:04,893 DEBUG [M:0;734d44df782b:40057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ebd3c166f79b4bb38069a917cc8eece1 is 52, key is load_balancer_on/state:d/1731270442210/Put/seqid=0 2024-11-10T20:28:04,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741900_1086 (size=5056) 2024-11-10T20:28:04,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741900_1086 (size=5056) 2024-11-10T20:28:04,900 INFO [M:0;734d44df782b:40057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ebd3c166f79b4bb38069a917cc8eece1 2024-11-10T20:28:04,906 DEBUG [M:0;734d44df782b:40057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/86224bebedb24b6cb011c98909ffc615 as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/86224bebedb24b6cb011c98909ffc615 2024-11-10T20:28:04,911 INFO [M:0;734d44df782b:40057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/86224bebedb24b6cb011c98909ffc615, entries=8, sequenceid=60, filesize=5.5 K 2024-11-10T20:28:04,912 DEBUG [M:0;734d44df782b:40057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb0dabc789334128bdbc056021d8e375 as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb0dabc789334128bdbc056021d8e375 2024-11-10T20:28:04,917 INFO [M:0;734d44df782b:40057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb0dabc789334128bdbc056021d8e375 2024-11-10T20:28:04,917 INFO [M:0;734d44df782b:40057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb0dabc789334128bdbc056021d8e375, entries=6, sequenceid=60, filesize=6.1 K 2024-11-10T20:28:04,918 DEBUG [M:0;734d44df782b:40057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb2bef8f1b6846b3a69e9e2f3be69c39 as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fb2bef8f1b6846b3a69e9e2f3be69c39 2024-11-10T20:28:04,923 INFO [M:0;734d44df782b:40057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fb2bef8f1b6846b3a69e9e2f3be69c39, entries=2, sequenceid=60, filesize=5.1 K 2024-11-10T20:28:04,924 DEBUG [M:0;734d44df782b:40057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ebd3c166f79b4bb38069a917cc8eece1 as hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ebd3c166f79b4bb38069a917cc8eece1 2024-11-10T20:28:04,929 INFO [M:0;734d44df782b:40057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ebd3c166f79b4bb38069a917cc8eece1, entries=1, sequenceid=60, filesize=4.9 K 2024-11-10T20:28:04,930 INFO [M:0;734d44df782b:40057 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 137ms, sequenceid=60, compaction requested=false 2024-11-10T20:28:04,931 INFO [M:0;734d44df782b:40057 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:04,932 DEBUG [M:0;734d44df782b:40057 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270484793Disabling compacts and flushes for region at 1731270484793Disabling writes for close at 1731270484793Obtaining lock to block concurrent updates at 1731270484793Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731270484793Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1731270484794 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731270484795 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731270484795Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731270484810 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731270484810Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731270484823 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731270484841 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731270484841Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731270484853 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731270484867 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731270484867Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731270484878 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731270484893 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731270484893Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7365f8f1: reopening flushed file at 1731270484905 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@132f4944: reopening flushed file at 1731270484911 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@181694ed: reopening flushed file at 1731270484917 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@424f2ba0: reopening flushed file at 1731270484923 (+6 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 137ms, sequenceid=60, compaction requested=false at 1731270484930 (+7 ms)Writing region close event to WAL at 1731270484931 (+1 ms)Closed at 1731270484931 2024-11-10T20:28:04,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:04,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46141 is added to blk_1073741880_1063 (size=1045) 2024-11-10T20:28:04,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39639 is added to blk_1073741880_1063 (size=1045) 2024-11-10T20:28:04,935 INFO [M:0;734d44df782b:40057 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T20:28:04,935 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:28:04,935 INFO [M:0;734d44df782b:40057 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40057 2024-11-10T20:28:04,935 INFO [M:0;734d44df782b:40057 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:28:05,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:05,038 INFO [M:0;734d44df782b:40057 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:28:05,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40057-0x10193ddee5a0000, quorum=127.0.0.1:55160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:05,040 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@633469fc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:05,041 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@28f4e296{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:05,041 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:05,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d04364e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:05,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@66e63d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:05,042 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:28:05,042 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:28:05,042 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:28:05,042 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@709d0d21 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:42605,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:38207 , LocalHost:localPort 734d44df782b/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-10T20:28:05,043 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1984011214-172.17.0.2-1731270440526 (Datanode Uuid 422d81f1-121b-44ef-a243-9c418ef1c7dc) service to localhost/127.0.0.1:34715 2024-11-10T20:28:05,043 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data3/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:05,044 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data4/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:05,044 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:28:05,044 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@709d0d21 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1984011214-172.17.0.2-1731270440526:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:42605,null,null], DatanodeInfoWithStorage[127.0.0.1:39639,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1984011214-172.17.0.2-1731270440526 2024-11-10T20:28:05,044 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@709d0d21 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:39639,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1984011214-172.17.0.2-1731270440526 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:05,044 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@709d0d21 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:42605,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1984011214-172.17.0.2-1731270440526 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:05,044 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@709d0d21 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:39639,null,null], DatanodeInfoWithStorage[127.0.0.1:42605,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1984011214-172.17.0.2-1731270440526:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:39639,null,null], DatanodeInfoWithStorage[127.0.0.1:42605,null,null]] 2024-11-10T20:28:05,046 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4438143d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:05,046 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32dac098{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:05,046 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:05,047 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@519de6b2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:05,047 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@83c4e47{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:05,048 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:28:05,048 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:28:05,048 WARN [BP-1984011214-172.17.0.2-1731270440526 heartbeating to localhost/127.0.0.1:34715 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1984011214-172.17.0.2-1731270440526 (Datanode Uuid 63e99a1c-a869-4ef8-9197-ea53d071eae7) service to localhost/127.0.0.1:34715 2024-11-10T20:28:05,048 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:28:05,049 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data7/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:05,049 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/cluster_2afc8b1f-7ff2-0d21-807c-21b83822a0a8/data/data8/current/BP-1984011214-172.17.0.2-1731270440526 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:05,049 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:28:05,055 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c00ef51{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:28:05,055 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@73b9709e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:05,055 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:05,055 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42b52d44{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:05,056 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@142d24a0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:05,064 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T20:28:05,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T20:28:05,101 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 79) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34715 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:38377 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34715 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38377 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:34715 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34715 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34715 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34715 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34715 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34715 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:34715 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007ff950bf46e8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34715 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007ff950bf46e8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:34715 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=452 (was 405) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=25 (was 10) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7429 (was 7862) 2024-11-10T20:28:05,109 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=25, ProcessCount=11, AvailableMemoryMB=7429 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.log.dir so I do NOT create it in target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8a1a04dc-a48f-06fa-3b59-154f621dc394/hadoop.tmp.dir so I do NOT create it in target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b, deleteOnExit=true 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/test.cache.data in system properties and HBase conf 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir in system properties and HBase conf 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T20:28:05,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T20:28:05,111 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T20:28:05,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:28:05,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:28:05,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T20:28:05,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:28:05,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T20:28:05,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T20:28:05,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:28:05,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:28:05,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T20:28:05,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/nfs.dump.dir in system properties and HBase conf 2024-11-10T20:28:05,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/java.io.tmpdir in system properties and HBase conf 2024-11-10T20:28:05,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:28:05,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T20:28:05,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T20:28:05,125 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:28:05,175 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:05,179 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:05,182 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:05,187 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:05,188 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:05,188 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:05,188 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:28:05,189 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:05,189 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a3130af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:05,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dbadaea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:05,303 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65e5bfc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/java.io.tmpdir/jetty-localhost-35201-hadoop-hdfs-3_4_1-tests_jar-_-any-11139183361965311276/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:28:05,304 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c23e5e9{HTTP/1.1, (http/1.1)}{localhost:35201} 2024-11-10T20:28:05,304 INFO [Time-limited test {}] server.Server(415): Started @147778ms 2024-11-10T20:28:05,317 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:28:05,375 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:05,380 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:05,383 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:05,383 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:05,383 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:28:05,384 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31880142{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:05,385 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de05dd9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:05,502 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7fd8c23{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/java.io.tmpdir/jetty-localhost-41825-hadoop-hdfs-3_4_1-tests_jar-_-any-17502206458004768785/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:05,503 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@77037455{HTTP/1.1, (http/1.1)}{localhost:41825} 2024-11-10T20:28:05,504 INFO [Time-limited test {}] server.Server(415): Started @147978ms 2024-11-10T20:28:05,506 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:28:05,538 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:05,542 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:05,543 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:05,543 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:05,543 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:28:05,544 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@624b3986{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:05,544 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@179d1ca6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:05,597 WARN [Thread-1183 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data1/current/BP-1789915910-172.17.0.2-1731270485133/current, will proceed with Du for space computation calculation, 2024-11-10T20:28:05,597 WARN [Thread-1184 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data2/current/BP-1789915910-172.17.0.2-1731270485133/current, will proceed with Du for space computation calculation, 2024-11-10T20:28:05,620 WARN [Thread-1162 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:28:05,623 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc5169d168927d9e6 with lease ID 0xcc92c66eeef5aa0e: Processing first storage report for DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727 from datanode DatanodeRegistration(127.0.0.1:41653, datanodeUuid=293800f1-564b-45e1-a843-e68e7f40dc6e, infoPort=46027, infoSecurePort=0, ipcPort=42219, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133) 2024-11-10T20:28:05,623 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc5169d168927d9e6 with lease ID 0xcc92c66eeef5aa0e: from storage DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727 node DatanodeRegistration(127.0.0.1:41653, datanodeUuid=293800f1-564b-45e1-a843-e68e7f40dc6e, infoPort=46027, infoSecurePort=0, ipcPort=42219, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:05,623 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc5169d168927d9e6 with lease ID 0xcc92c66eeef5aa0e: Processing first storage report for DS-f3b36a97-e81c-43be-8f0b-4bc71132fefd from datanode DatanodeRegistration(127.0.0.1:41653, datanodeUuid=293800f1-564b-45e1-a843-e68e7f40dc6e, infoPort=46027, infoSecurePort=0, ipcPort=42219, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133) 2024-11-10T20:28:05,623 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc5169d168927d9e6 with lease ID 0xcc92c66eeef5aa0e: from storage DS-f3b36a97-e81c-43be-8f0b-4bc71132fefd node DatanodeRegistration(127.0.0.1:41653, datanodeUuid=293800f1-564b-45e1-a843-e68e7f40dc6e, infoPort=46027, infoSecurePort=0, ipcPort=42219, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:05,675 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a9fc515{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/java.io.tmpdir/jetty-localhost-45269-hadoop-hdfs-3_4_1-tests_jar-_-any-15057738781630458326/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:05,676 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@38a385f1{HTTP/1.1, (http/1.1)}{localhost:45269} 2024-11-10T20:28:05,676 INFO [Time-limited test {}] server.Server(415): Started @148150ms 2024-11-10T20:28:05,677 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:28:05,759 WARN [Thread-1209 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data3/current/BP-1789915910-172.17.0.2-1731270485133/current, will proceed with Du for space computation calculation, 2024-11-10T20:28:05,759 WARN [Thread-1210 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data4/current/BP-1789915910-172.17.0.2-1731270485133/current, will proceed with Du for space computation calculation, 2024-11-10T20:28:05,775 WARN [Thread-1198 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:28:05,777 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe601d00d9c2d9976 with lease ID 0xcc92c66eeef5aa0f: Processing first storage report for DS-6160c879-2f46-465a-82f3-76e411f22609 from datanode DatanodeRegistration(127.0.0.1:41959, datanodeUuid=8debfdad-9a14-4393-bc0e-9cd36739f6dd, infoPort=37359, infoSecurePort=0, ipcPort=45869, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133) 2024-11-10T20:28:05,777 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe601d00d9c2d9976 with lease ID 0xcc92c66eeef5aa0f: from storage DS-6160c879-2f46-465a-82f3-76e411f22609 node DatanodeRegistration(127.0.0.1:41959, datanodeUuid=8debfdad-9a14-4393-bc0e-9cd36739f6dd, infoPort=37359, infoSecurePort=0, ipcPort=45869, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:05,778 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe601d00d9c2d9976 with lease ID 0xcc92c66eeef5aa0f: Processing first storage report for DS-0ec36b5d-b380-49be-a594-fee55d3d75d1 from datanode DatanodeRegistration(127.0.0.1:41959, datanodeUuid=8debfdad-9a14-4393-bc0e-9cd36739f6dd, infoPort=37359, infoSecurePort=0, ipcPort=45869, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133) 2024-11-10T20:28:05,778 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe601d00d9c2d9976 with lease ID 0xcc92c66eeef5aa0f: from storage DS-0ec36b5d-b380-49be-a594-fee55d3d75d1 node DatanodeRegistration(127.0.0.1:41959, datanodeUuid=8debfdad-9a14-4393-bc0e-9cd36739f6dd, infoPort=37359, infoSecurePort=0, ipcPort=45869, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:05,800 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03 2024-11-10T20:28:05,803 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/zookeeper_0, clientPort=59131, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T20:28:05,804 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59131 2024-11-10T20:28:05,804 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:05,806 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:05,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:28:05,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:28:05,816 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504 with version=8 2024-11-10T20:28:05,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/hbase-staging 2024-11-10T20:28:05,818 INFO [Time-limited test {}] client.ConnectionUtils(128): master/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:28:05,818 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:05,818 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:05,818 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:28:05,818 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:05,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:28:05,819 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T20:28:05,819 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:28:05,820 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43943 2024-11-10T20:28:05,821 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43943 connecting to ZooKeeper ensemble=127.0.0.1:59131 2024-11-10T20:28:05,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:439430x0, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:28:05,829 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43943-0x10193de9c930000 connected 2024-11-10T20:28:05,844 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:05,845 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:05,847 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:28:05,847 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504, hbase.cluster.distributed=false 2024-11-10T20:28:05,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:28:05,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43943 2024-11-10T20:28:05,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43943 2024-11-10T20:28:05,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43943 2024-11-10T20:28:05,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43943 2024-11-10T20:28:05,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43943 2024-11-10T20:28:05,865 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:28:05,865 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:05,865 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:05,865 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:28:05,865 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:05,865 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:28:05,866 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T20:28:05,866 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:28:05,866 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33961 2024-11-10T20:28:05,867 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33961 connecting to ZooKeeper ensemble=127.0.0.1:59131 2024-11-10T20:28:05,868 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:05,870 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:05,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:339610x0, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:28:05,875 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:339610x0, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:28:05,875 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33961-0x10193de9c930001 connected 2024-11-10T20:28:05,875 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T20:28:05,876 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T20:28:05,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T20:28:05,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:28:05,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33961 2024-11-10T20:28:05,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33961 2024-11-10T20:28:05,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33961 2024-11-10T20:28:05,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33961 2024-11-10T20:28:05,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33961 2024-11-10T20:28:05,890 DEBUG [M:0;734d44df782b:43943 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;734d44df782b:43943 2024-11-10T20:28:05,891 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/734d44df782b,43943,1731270485818 2024-11-10T20:28:05,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:28:05,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:28:05,893 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/734d44df782b,43943,1731270485818 2024-11-10T20:28:05,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T20:28:05,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:05,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:05,896 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T20:28:05,896 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/734d44df782b,43943,1731270485818 from backup master directory 2024-11-10T20:28:05,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/734d44df782b,43943,1731270485818 2024-11-10T20:28:05,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:28:05,898 WARN [master/734d44df782b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:28:05,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:28:05,898 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=734d44df782b,43943,1731270485818 2024-11-10T20:28:05,902 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/hbase.id] with ID: 942c904f-4b37-4d69-a876-91dfc1994811 2024-11-10T20:28:05,902 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/.tmp/hbase.id 2024-11-10T20:28:05,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:28:05,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:28:05,909 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/.tmp/hbase.id]:[hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/hbase.id] 2024-11-10T20:28:05,920 INFO [master/734d44df782b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:05,921 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T20:28:05,922 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T20:28:05,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:05,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:05,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:28:05,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:28:05,931 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:28:05,932 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T20:28:05,932 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:28:05,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:28:05,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:28:05,940 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store 2024-11-10T20:28:05,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:28:05,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:28:05,948 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:05,948 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:28:05,948 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:05,948 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:05,948 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:28:05,948 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:05,948 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:05,948 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270485948Disabling compacts and flushes for region at 1731270485948Disabling writes for close at 1731270485948Writing region close event to WAL at 1731270485948Closed at 1731270485948 2024-11-10T20:28:05,949 WARN [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/.initializing 2024-11-10T20:28:05,949 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818 2024-11-10T20:28:05,951 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C43943%2C1731270485818, suffix=, logDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818, archiveDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/oldWALs, maxLogs=10 2024-11-10T20:28:05,952 INFO [master/734d44df782b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C43943%2C1731270485818.1731270485951 2024-11-10T20:28:05,956 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270485951 2024-11-10T20:28:05,957 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37359:37359),(127.0.0.1/127.0.0.1:46027:46027)] 2024-11-10T20:28:05,957 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:28:05,958 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:05,958 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,958 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,960 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T20:28:05,960 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:05,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:05,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T20:28:05,962 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:05,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:28:05,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,963 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T20:28:05,963 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:05,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:28:05,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T20:28:05,965 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:05,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:28:05,965 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,966 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,966 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,967 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,967 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,968 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T20:28:05,969 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:05,971 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:28:05,971 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=790463, jitterRate=0.005126252770423889}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T20:28:05,972 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731270485958Initializing all the Stores at 1731270485959 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270485959Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270485959Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270485959Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270485959Cleaning up temporary data from old regions at 1731270485967 (+8 ms)Region opened successfully at 1731270485972 (+5 ms) 2024-11-10T20:28:05,973 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T20:28:05,976 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f5973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:28:05,977 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T20:28:05,977 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T20:28:05,977 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T20:28:05,977 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T20:28:05,978 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T20:28:05,978 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T20:28:05,978 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T20:28:05,980 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T20:28:05,980 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T20:28:05,982 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T20:28:05,982 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T20:28:05,983 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T20:28:05,984 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T20:28:05,984 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T20:28:05,985 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T20:28:05,987 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T20:28:05,988 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T20:28:05,990 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T20:28:05,992 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T20:28:05,994 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T20:28:05,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:28:05,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:28:05,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:05,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:05,997 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=734d44df782b,43943,1731270485818, sessionid=0x10193de9c930000, setting cluster-up flag (Was=false) 2024-11-10T20:28:06,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:06,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:06,006 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T20:28:06,007 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,43943,1731270485818 2024-11-10T20:28:06,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:06,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:06,016 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T20:28:06,017 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,43943,1731270485818 2024-11-10T20:28:06,018 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T20:28:06,020 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T20:28:06,021 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T20:28:06,021 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T20:28:06,021 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 734d44df782b,43943,1731270485818 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T20:28:06,022 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:28:06,022 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:28:06,023 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:28:06,023 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:28:06,023 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/734d44df782b:0, corePoolSize=10, maxPoolSize=10 2024-11-10T20:28:06,023 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,023 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:28:06,023 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,025 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:28:06,025 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T20:28:06,026 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,026 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T20:28:06,029 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731270516029 2024-11-10T20:28:06,029 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T20:28:06,029 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T20:28:06,029 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T20:28:06,029 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T20:28:06,029 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T20:28:06,029 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T20:28:06,031 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,032 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T20:28:06,032 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T20:28:06,032 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T20:28:06,032 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T20:28:06,033 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T20:28:06,033 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270486033,5,FailOnTimeoutGroup] 2024-11-10T20:28:06,033 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270486033,5,FailOnTimeoutGroup] 2024-11-10T20:28:06,033 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,033 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T20:28:06,033 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,033 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:28:06,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:28:06,039 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T20:28:06,039 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504 2024-11-10T20:28:06,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:28:06,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:28:06,049 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:06,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:28:06,052 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:28:06,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:06,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:28:06,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:28:06,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:06,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:28:06,056 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:28:06,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:06,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:28:06,058 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:28:06,058 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:06,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:28:06,060 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740 2024-11-10T20:28:06,060 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740 2024-11-10T20:28:06,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:28:06,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:28:06,062 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:28:06,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:28:06,065 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:28:06,065 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=741399, jitterRate=-0.05726364254951477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:28:06,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731270486049Initializing all the Stores at 1731270486050 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270486050Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270486051 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270486051Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270486051Cleaning up temporary data from old regions at 1731270486061 (+10 ms)Region opened successfully at 1731270486066 (+5 ms) 2024-11-10T20:28:06,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:28:06,066 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:28:06,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:28:06,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:28:06,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:28:06,067 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:28:06,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270486066Disabling compacts and flushes for region at 1731270486066Disabling writes for close at 1731270486066Writing region close event to WAL at 1731270486067 (+1 ms)Closed at 1731270486067 2024-11-10T20:28:06,068 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:28:06,068 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T20:28:06,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T20:28:06,069 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:28:06,070 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T20:28:06,080 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(746): ClusterId : 942c904f-4b37-4d69-a876-91dfc1994811 2024-11-10T20:28:06,080 DEBUG [RS:0;734d44df782b:33961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T20:28:06,083 DEBUG [RS:0;734d44df782b:33961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T20:28:06,083 DEBUG [RS:0;734d44df782b:33961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T20:28:06,087 DEBUG [RS:0;734d44df782b:33961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T20:28:06,087 DEBUG [RS:0;734d44df782b:33961 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bafed9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:28:06,099 DEBUG [RS:0;734d44df782b:33961 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;734d44df782b:33961 2024-11-10T20:28:06,099 INFO [RS:0;734d44df782b:33961 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T20:28:06,099 INFO [RS:0;734d44df782b:33961 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T20:28:06,099 DEBUG [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T20:28:06,100 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(2659): reportForDuty to master=734d44df782b,43943,1731270485818 with port=33961, startcode=1731270485865 2024-11-10T20:28:06,100 DEBUG [RS:0;734d44df782b:33961 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T20:28:06,102 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57395, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T20:28:06,102 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43943 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 734d44df782b,33961,1731270485865 2024-11-10T20:28:06,102 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43943 {}] master.ServerManager(517): Registering regionserver=734d44df782b,33961,1731270485865 2024-11-10T20:28:06,104 DEBUG [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504 2024-11-10T20:28:06,104 DEBUG [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37029 2024-11-10T20:28:06,104 DEBUG [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T20:28:06,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:28:06,107 DEBUG [RS:0;734d44df782b:33961 {}] zookeeper.ZKUtil(111): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/734d44df782b,33961,1731270485865 2024-11-10T20:28:06,107 WARN [RS:0;734d44df782b:33961 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:28:06,107 INFO [RS:0;734d44df782b:33961 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:28:06,107 DEBUG [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865 2024-11-10T20:28:06,107 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [734d44df782b,33961,1731270485865] 2024-11-10T20:28:06,111 INFO [RS:0;734d44df782b:33961 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T20:28:06,112 INFO [RS:0;734d44df782b:33961 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T20:28:06,113 INFO [RS:0;734d44df782b:33961 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T20:28:06,113 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,113 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T20:28:06,114 INFO [RS:0;734d44df782b:33961 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T20:28:06,114 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:28:06,114 DEBUG [RS:0;734d44df782b:33961 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:28:06,119 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,119 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,119 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,119 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,119 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,119 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33961,1731270485865-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:28:06,134 INFO [RS:0;734d44df782b:33961 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T20:28:06,134 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33961,1731270485865-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,134 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,134 INFO [RS:0;734d44df782b:33961 {}] regionserver.Replication(171): 734d44df782b,33961,1731270485865 started 2024-11-10T20:28:06,148 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,148 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(1482): Serving as 734d44df782b,33961,1731270485865, RpcServer on 734d44df782b/172.17.0.2:33961, sessionid=0x10193de9c930001 2024-11-10T20:28:06,149 DEBUG [RS:0;734d44df782b:33961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T20:28:06,149 DEBUG [RS:0;734d44df782b:33961 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 734d44df782b,33961,1731270485865 2024-11-10T20:28:06,149 DEBUG [RS:0;734d44df782b:33961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,33961,1731270485865' 2024-11-10T20:28:06,149 DEBUG [RS:0;734d44df782b:33961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T20:28:06,149 DEBUG [RS:0;734d44df782b:33961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T20:28:06,150 DEBUG [RS:0;734d44df782b:33961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T20:28:06,150 DEBUG [RS:0;734d44df782b:33961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T20:28:06,150 DEBUG [RS:0;734d44df782b:33961 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 734d44df782b,33961,1731270485865 2024-11-10T20:28:06,150 DEBUG [RS:0;734d44df782b:33961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,33961,1731270485865' 2024-11-10T20:28:06,150 DEBUG [RS:0;734d44df782b:33961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T20:28:06,151 DEBUG [RS:0;734d44df782b:33961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T20:28:06,151 DEBUG [RS:0;734d44df782b:33961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T20:28:06,151 INFO [RS:0;734d44df782b:33961 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T20:28:06,151 INFO [RS:0;734d44df782b:33961 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T20:28:06,176 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:06,180 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:06,221 WARN [734d44df782b:43943 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T20:28:06,253 INFO [RS:0;734d44df782b:33961 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C33961%2C1731270485865, suffix=, logDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865, archiveDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/oldWALs, maxLogs=32 2024-11-10T20:28:06,254 INFO [RS:0;734d44df782b:33961 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33961%2C1731270485865.1731270486254 2024-11-10T20:28:06,261 INFO [RS:0;734d44df782b:33961 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 2024-11-10T20:28:06,262 DEBUG [RS:0;734d44df782b:33961 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37359:37359),(127.0.0.1/127.0.0.1:46027:46027)] 2024-11-10T20:28:06,471 DEBUG [734d44df782b:43943 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T20:28:06,471 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=734d44df782b,33961,1731270485865 2024-11-10T20:28:06,473 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,33961,1731270485865, state=OPENING 2024-11-10T20:28:06,475 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T20:28:06,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:06,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:06,477 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:28:06,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,33961,1731270485865}] 2024-11-10T20:28:06,478 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:28:06,478 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:28:06,631 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T20:28:06,633 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45915, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T20:28:06,636 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T20:28:06,636 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:28:06,638 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C33961%2C1731270485865.meta, suffix=.meta, logDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865, archiveDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/oldWALs, maxLogs=32 2024-11-10T20:28:06,639 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta 2024-11-10T20:28:06,644 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta 2024-11-10T20:28:06,645 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46027:46027),(127.0.0.1/127.0.0.1:37359:37359)] 2024-11-10T20:28:06,645 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:28:06,646 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T20:28:06,646 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T20:28:06,646 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T20:28:06,646 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T20:28:06,646 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:06,646 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T20:28:06,646 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T20:28:06,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:28:06,649 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:28:06,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:06,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:28:06,650 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:28:06,650 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:06,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:28:06,652 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:28:06,652 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:06,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:28:06,653 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:28:06,653 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:06,654 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:28:06,654 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740 2024-11-10T20:28:06,656 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740 2024-11-10T20:28:06,657 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:28:06,657 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:28:06,658 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:28:06,659 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:28:06,660 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=829970, jitterRate=0.05536144971847534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:28:06,660 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T20:28:06,661 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731270486647Writing region info on filesystem at 1731270486647Initializing all the Stores at 1731270486647Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270486648 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270486648Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270486648Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270486648Cleaning up temporary data from old regions at 1731270486657 (+9 ms)Running coprocessor post-open hooks at 1731270486660 (+3 ms)Region opened successfully at 1731270486661 (+1 ms) 2024-11-10T20:28:06,662 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731270486631 2024-11-10T20:28:06,664 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T20:28:06,664 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T20:28:06,665 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,33961,1731270485865 2024-11-10T20:28:06,666 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,33961,1731270485865, state=OPEN 2024-11-10T20:28:06,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:28:06,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:28:06,671 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=734d44df782b,33961,1731270485865 2024-11-10T20:28:06,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:28:06,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:28:06,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T20:28:06,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,33961,1731270485865 in 193 msec 2024-11-10T20:28:06,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T20:28:06,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec 2024-11-10T20:28:06,678 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:28:06,678 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T20:28:06,679 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:28:06,680 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,33961,1731270485865, seqNum=-1] 2024-11-10T20:28:06,680 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:28:06,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48771, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:28:06,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 666 msec 2024-11-10T20:28:06,687 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731270486687, completionTime=-1 2024-11-10T20:28:06,687 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T20:28:06,687 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T20:28:06,688 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T20:28:06,688 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731270546688 2024-11-10T20:28:06,689 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731270606689 2024-11-10T20:28:06,689 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-10T20:28:06,689 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43943,1731270485818-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,689 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43943,1731270485818-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,689 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43943,1731270485818-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,689 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-734d44df782b:43943, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,689 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,689 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,691 DEBUG [master/734d44df782b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T20:28:06,693 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.795sec 2024-11-10T20:28:06,693 INFO [master/734d44df782b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T20:28:06,693 INFO [master/734d44df782b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T20:28:06,693 INFO [master/734d44df782b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T20:28:06,693 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T20:28:06,693 INFO [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T20:28:06,693 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43943,1731270485818-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:28:06,693 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43943,1731270485818-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T20:28:06,695 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T20:28:06,695 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T20:28:06,695 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,43943,1731270485818-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:06,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7f214b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:28:06,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 734d44df782b,43943,-1 for getting cluster id 2024-11-10T20:28:06,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T20:28:06,783 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '942c904f-4b37-4d69-a876-91dfc1994811' 2024-11-10T20:28:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T20:28:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "942c904f-4b37-4d69-a876-91dfc1994811" 2024-11-10T20:28:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce289d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:28:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [734d44df782b,43943,-1] 2024-11-10T20:28:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T20:28:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:28:06,785 INFO [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T20:28:06,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40375d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:28:06,786 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:28:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,33961,1731270485865, seqNum=-1] 2024-11-10T20:28:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:28:06,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:28:06,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=734d44df782b,43943,1731270485818 2024-11-10T20:28:06,791 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:06,793 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T20:28:06,793 INFO [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart 2024-11-10T20:28:06,793 INFO [Time-limited test {}] wal.TestLogRolling(323): Replication=2 2024-11-10T20:28:06,794 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T20:28:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 734d44df782b,43943,1731270485818 2024-11-10T20:28:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@77dc3d81 2024-11-10T20:28:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T20:28:06,796 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T20:28:06,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43943 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T20:28:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T20:28:06,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43943 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:28:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43943 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T20:28:06,799 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T20:28:06,800 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:06,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43943 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 4 2024-11-10T20:28:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:28:06,801 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T20:28:06,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741835_1011 (size=395) 2024-11-10T20:28:06,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741835_1011 (size=395) 2024-11-10T20:28:06,809 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1f3a13100bb802bf8b0a70eafbc11e37, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504 2024-11-10T20:28:06,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741836_1012 (size=78) 2024-11-10T20:28:06,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41653 is added to blk_1073741836_1012 (size=78) 2024-11-10T20:28:06,816 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:06,816 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 1f3a13100bb802bf8b0a70eafbc11e37, disabling compactions & flushes 2024-11-10T20:28:06,816 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:06,816 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:06,816 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. after waiting 0 ms 2024-11-10T20:28:06,816 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:06,816 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:06,816 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1f3a13100bb802bf8b0a70eafbc11e37: Waiting for close lock at 1731270486816Disabling compacts and flushes for region at 1731270486816Disabling writes for close at 1731270486816Writing region close event to WAL at 1731270486816Closed at 1731270486816 2024-11-10T20:28:06,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T20:28:06,818 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731270486818"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731270486818"}]},"ts":"1731270486818"} 2024-11-10T20:28:06,821 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T20:28:06,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T20:28:06,822 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270486822"}]},"ts":"1731270486822"} 2024-11-10T20:28:06,824 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-11-10T20:28:06,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1f3a13100bb802bf8b0a70eafbc11e37, ASSIGN}] 2024-11-10T20:28:06,826 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1f3a13100bb802bf8b0a70eafbc11e37, ASSIGN 2024-11-10T20:28:06,827 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1f3a13100bb802bf8b0a70eafbc11e37, ASSIGN; state=OFFLINE, location=734d44df782b,33961,1731270485865; forceNewPlan=false, retain=false 2024-11-10T20:28:06,977 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1f3a13100bb802bf8b0a70eafbc11e37, regionState=OPENING, regionLocation=734d44df782b,33961,1731270485865 2024-11-10T20:28:06,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1f3a13100bb802bf8b0a70eafbc11e37, ASSIGN because future has completed 2024-11-10T20:28:06,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f3a13100bb802bf8b0a70eafbc11e37, server=734d44df782b,33961,1731270485865}] 2024-11-10T20:28:07,138 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:07,138 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f3a13100bb802bf8b0a70eafbc11e37, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:28:07,138 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,138 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:07,138 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,138 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,140 INFO [StoreOpener-1f3a13100bb802bf8b0a70eafbc11e37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,141 INFO [StoreOpener-1f3a13100bb802bf8b0a70eafbc11e37-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f3a13100bb802bf8b0a70eafbc11e37 columnFamilyName info 2024-11-10T20:28:07,141 DEBUG [StoreOpener-1f3a13100bb802bf8b0a70eafbc11e37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:07,142 INFO [StoreOpener-1f3a13100bb802bf8b0a70eafbc11e37-1 {}] regionserver.HStore(327): Store=1f3a13100bb802bf8b0a70eafbc11e37/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:28:07,142 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,143 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/default/TestLogRolling-testLogRollOnPipelineRestart/1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,143 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/default/TestLogRolling-testLogRollOnPipelineRestart/1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,144 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,144 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,145 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,147 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/default/TestLogRolling-testLogRollOnPipelineRestart/1f3a13100bb802bf8b0a70eafbc11e37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:28:07,148 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1f3a13100bb802bf8b0a70eafbc11e37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=786463, jitterRate=3.986060619354248E-5}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T20:28:07,148 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:07,149 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1f3a13100bb802bf8b0a70eafbc11e37: Running coprocessor pre-open hook at 1731270487138Writing region info on filesystem at 1731270487138Initializing all the Stores at 1731270487139 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270487139Cleaning up temporary data from old regions at 1731270487144 (+5 ms)Running coprocessor post-open hooks at 1731270487148 (+4 ms)Region opened successfully at 1731270487149 (+1 ms) 2024-11-10T20:28:07,150 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37., pid=6, masterSystemTime=1731270487134 2024-11-10T20:28:07,153 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:07,153 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:07,154 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1f3a13100bb802bf8b0a70eafbc11e37, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,33961,1731270485865 2024-11-10T20:28:07,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f3a13100bb802bf8b0a70eafbc11e37, server=734d44df782b,33961,1731270485865 because future has completed 2024-11-10T20:28:07,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T20:28:07,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f3a13100bb802bf8b0a70eafbc11e37, server=734d44df782b,33961,1731270485865 in 177 msec 2024-11-10T20:28:07,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T20:28:07,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=1f3a13100bb802bf8b0a70eafbc11e37, ASSIGN in 337 msec 2024-11-10T20:28:07,165 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T20:28:07,165 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270487165"}]},"ts":"1731270487165"} 2024-11-10T20:28:07,167 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-11-10T20:28:07,168 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T20:28:07,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 371 msec 2024-11-10T20:28:07,177 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:07,180 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:08,177 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:08,181 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:09,178 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:09,181 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:10,178 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:10,182 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:11,179 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:11,182 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:12,148 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T20:28:12,163 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,163 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,164 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,164 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,165 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,165 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,168 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,169 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,169 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,171 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:12,177 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T20:28:12,177 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-11-10T20:28:12,180 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:12,183 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:12,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T20:28:12,804 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T20:28:12,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T20:28:12,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-11-10T20:28:12,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:28:12,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T20:28:13,180 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:13,183 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:14,181 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:14,184 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:15,181 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:15,184 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:16,182 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:16,185 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:28:16,862 INFO [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed 2024-11-10T20:28:16,863 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100 2024-11-10T20:28:16,866 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T20:28:16,866 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37., hostname=734d44df782b,33961,1731270485865, seqNum=2] 2024-11-10T20:28:17,183 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:17,185 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:18,183 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:18,186 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:18,872 INFO [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 2024-11-10T20:28:18,872 WARN [ResponseProcessor for block BP-1789915910-172.17.0.2-1731270485133:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1789915910-172.17.0.2-1731270485133:blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:18,872 WARN [ResponseProcessor for block BP-1789915910-172.17.0.2-1731270485133:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1789915910-172.17.0.2-1731270485133:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:18,873 WARN [ResponseProcessor for block BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:41959,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:18,873 WARN [DataStreamer for file /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270485951 block BP-1789915910-172.17.0.2-1731270485133:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1789915910-172.17.0.2-1731270485133:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41959,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK], DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41959,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]) is bad. 2024-11-10T20:28:18,873 WARN [DataStreamer for file /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 block BP-1789915910-172.17.0.2-1731270485133:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1789915910-172.17.0.2-1731270485133:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41959,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK], DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41959,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]) is bad. 2024-11-10T20:28:18,873 WARN [DataStreamer for file /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta block BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK], DatanodeInfoWithStorage[127.0.0.1:41959,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41959,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]) is bad. 2024-11-10T20:28:18,873 WARN [PacketResponder: BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41959] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:18,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711254180_22 at /127.0.0.1:42950 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42950 dst: /127.0.0.1:41653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:18,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711254180_22 at /127.0.0.1:51342 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51342 dst: /127.0.0.1:41959 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:18,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711254180_22 at /127.0.0.1:42938 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42938 dst: /127.0.0.1:41653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:18,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711254180_22 at /127.0.0.1:51344 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51344 dst: /127.0.0.1:41959 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:18,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1515544807_22 at /127.0.0.1:51332 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51332 dst: /127.0.0.1:41959 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:18,875 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1515544807_22 at /127.0.0.1:42910 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42910 dst: /127.0.0.1:41653 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:18,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a9fc515{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:18,881 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@38a385f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:18,881 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:18,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@179d1ca6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:18,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@624b3986{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:18,883 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:28:18,883 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1789915910-172.17.0.2-1731270485133 (Datanode Uuid 8debfdad-9a14-4393-bc0e-9cd36739f6dd) service to localhost/127.0.0.1:37029 2024-11-10T20:28:18,883 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:28:18,883 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:28:18,883 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data3/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:18,884 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data4/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:18,884 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:28:18,893 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:18,896 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:18,897 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:18,897 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:18,897 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:28:18,897 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5507dc7e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:18,898 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@27e9dc43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:19,012 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7aa94e72{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/java.io.tmpdir/jetty-localhost-41665-hadoop-hdfs-3_4_1-tests_jar-_-any-9525901148736318039/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:19,012 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@f09ac85{HTTP/1.1, (http/1.1)}{localhost:41665} 2024-11-10T20:28:19,012 INFO [Time-limited test {}] server.Server(415): Started @161486ms 2024-11-10T20:28:19,013 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:28:19,033 WARN [ResponseProcessor for block BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:19,033 WARN [ResponseProcessor for block BP-1789915910-172.17.0.2-1731270485133:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1789915910-172.17.0.2-1731270485133:blk_1073741830_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:19,033 WARN [ResponseProcessor for block BP-1789915910-172.17.0.2-1731270485133:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1789915910-172.17.0.2-1731270485133:blk_1073741833_1014 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:19,034 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1515544807_22 at /127.0.0.1:57582 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57582 dst: /127.0.0.1:41653 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:19,034 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711254180_22 at /127.0.0.1:57596 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57596 dst: /127.0.0.1:41653 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:19,035 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711254180_22 at /127.0.0.1:57612 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41653:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57612 dst: /127.0.0.1:41653 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:19,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7fd8c23{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:19,041 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@77037455{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:19,041 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:19,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de05dd9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:19,042 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31880142{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:19,043 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:28:19,043 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:28:19,043 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1789915910-172.17.0.2-1731270485133 (Datanode Uuid 293800f1-564b-45e1-a843-e68e7f40dc6e) service to localhost/127.0.0.1:37029 2024-11-10T20:28:19,043 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:28:19,044 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data1/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:19,045 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data2/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:19,045 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:28:19,053 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:19,056 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:19,056 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:19,056 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:19,057 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:28:19,057 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a7a4b6c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:19,057 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f3a1cd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:19,104 WARN [Thread-1333 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:28:19,106 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xffb582aa0a806fe1 with lease ID 0xcc92c66eeef5aa10: from storage DS-6160c879-2f46-465a-82f3-76e411f22609 node DatanodeRegistration(127.0.0.1:38393, datanodeUuid=8debfdad-9a14-4393-bc0e-9cd36739f6dd, infoPort=39283, infoSecurePort=0, ipcPort=44405, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:19,107 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xffb582aa0a806fe1 with lease ID 0xcc92c66eeef5aa10: from storage DS-0ec36b5d-b380-49be-a594-fee55d3d75d1 node DatanodeRegistration(127.0.0.1:38393, datanodeUuid=8debfdad-9a14-4393-bc0e-9cd36739f6dd, infoPort=39283, infoSecurePort=0, ipcPort=44405, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:19,176 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3903d405{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/java.io.tmpdir/jetty-localhost-41409-hadoop-hdfs-3_4_1-tests_jar-_-any-9175234039485412368/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:19,177 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3977d0c{HTTP/1.1, (http/1.1)}{localhost:41409} 2024-11-10T20:28:19,177 INFO [Time-limited test {}] server.Server(415): Started @161651ms 2024-11-10T20:28:19,178 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:28:19,184 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:19,186 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:19,294 WARN [Thread-1364 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:28:19,296 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6989ef54e44b5dc0 with lease ID 0xcc92c66eeef5aa11: from storage DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727 node DatanodeRegistration(127.0.0.1:40045, datanodeUuid=293800f1-564b-45e1-a843-e68e7f40dc6e, infoPort=37187, infoSecurePort=0, ipcPort=36185, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:19,297 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6989ef54e44b5dc0 with lease ID 0xcc92c66eeef5aa11: from storage DS-f3b36a97-e81c-43be-8f0b-4bc71132fefd node DatanodeRegistration(127.0.0.1:40045, datanodeUuid=293800f1-564b-45e1-a843-e68e7f40dc6e, infoPort=37187, infoSecurePort=0, ipcPort=36185, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:20,184 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:20,187 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:20,198 INFO [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted 2024-11-10T20:28:20,201 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-11-10T20:28:20,202 ERROR [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504-prefix:734d44df782b,33961,1731270485865 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:20,202 WARN [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504-prefix:734d44df782b,33961,1731270485865 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:20,203 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C33961%2C1731270485865:(num 1731270486254) roll requested 2024-11-10T20:28:20,203 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33961%2C1731270485865.1731270500203 2024-11-10T20:28:20,209 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 newFile=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 2024-11-10T20:28:20,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:20,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:20,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:20,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:20,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:20,209 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 2024-11-10T20:28:20,210 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:20,210 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:20,210 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 2024-11-10T20:28:20,210 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37187:37187),(127.0.0.1/127.0.0.1:39283:39283)] 2024-11-10T20:28:20,210 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 is not closed yet, will try archiving it next time 2024-11-10T20:28:20,211 WARN [IPC Server handler 4 on default port 37029 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014 2024-11-10T20:28:20,211 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 after 1ms 2024-11-10T20:28:21,108 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T20:28:21,185 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:21,187 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:22,186 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:22,188 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:22,214 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-11-10T20:28:23,186 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:23,188 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:24,187 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:24,189 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:24,212 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 after 4002ms 2024-11-10T20:28:24,217 WARN [ResponseProcessor for block BP-1789915910-172.17.0.2-1731270485133:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1789915910-172.17.0.2-1731270485133:blk_1073741837_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:24,217 WARN [DataStreamer for file /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 block BP-1789915910-172.17.0.2-1731270485133:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1789915910-172.17.0.2-1731270485133:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40045,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK], DatanodeInfoWithStorage[127.0.0.1:38393,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40045,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]) is bad. 2024-11-10T20:28:24,218 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711254180_22 at /127.0.0.1:42682 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38393:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42682 dst: /127.0.0.1:38393 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:24,218 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711254180_22 at /127.0.0.1:34634 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:40045:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34634 dst: /127.0.0.1:40045 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:24,219 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3903d405{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:24,220 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3977d0c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:24,220 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:24,220 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12f3a1cd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:24,220 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a7a4b6c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:24,221 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:28:24,221 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1789915910-172.17.0.2-1731270485133 (Datanode Uuid 293800f1-564b-45e1-a843-e68e7f40dc6e) service to localhost/127.0.0.1:37029 2024-11-10T20:28:24,221 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:28:24,221 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:28:24,222 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data1/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:24,223 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data2/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:24,223 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:28:24,230 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:24,233 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:24,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:24,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:24,234 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:28:24,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@717499fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:24,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62802d30{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:24,348 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1dcfabbb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/java.io.tmpdir/jetty-localhost-36883-hadoop-hdfs-3_4_1-tests_jar-_-any-18350739121731032422/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:24,348 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6614137b{HTTP/1.1, (http/1.1)}{localhost:36883} 2024-11-10T20:28:24,348 INFO [Time-limited test {}] server.Server(415): Started @166822ms 2024-11-10T20:28:24,349 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:28:24,366 WARN [ResponseProcessor for block BP-1789915910-172.17.0.2-1731270485133:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1789915910-172.17.0.2-1731270485133:blk_1073741837_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:24,366 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711254180_22 at /127.0.0.1:42704 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38393:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42704 dst: /127.0.0.1:38393 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:24,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7aa94e72{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:24,368 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@f09ac85{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:24,368 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:24,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@27e9dc43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:24,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5507dc7e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:24,369 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:28:24,369 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:28:24,369 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1789915910-172.17.0.2-1731270485133 (Datanode Uuid 8debfdad-9a14-4393-bc0e-9cd36739f6dd) service to localhost/127.0.0.1:37029 2024-11-10T20:28:24,369 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:28:24,370 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data3/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:24,370 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data4/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:24,371 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:28:24,385 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:24,389 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:24,390 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:24,390 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:24,390 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:28:24,391 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3146549d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:24,391 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6baabd83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:24,443 WARN [Thread-1407 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:28:24,446 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc004e8750ecef34 with lease ID 0xcc92c66eeef5aa12: from storage DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727 node DatanodeRegistration(127.0.0.1:41867, datanodeUuid=293800f1-564b-45e1-a843-e68e7f40dc6e, infoPort=32943, infoSecurePort=0, ipcPort=34891, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:24,446 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc004e8750ecef34 with lease ID 0xcc92c66eeef5aa12: from storage DS-f3b36a97-e81c-43be-8f0b-4bc71132fefd node DatanodeRegistration(127.0.0.1:41867, datanodeUuid=293800f1-564b-45e1-a843-e68e7f40dc6e, infoPort=32943, infoSecurePort=0, ipcPort=34891, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:24,506 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21404da7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/java.io.tmpdir/jetty-localhost-37543-hadoop-hdfs-3_4_1-tests_jar-_-any-3046324866664651234/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:24,507 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e8914c8{HTTP/1.1, (http/1.1)}{localhost:37543} 2024-11-10T20:28:24,507 INFO [Time-limited test {}] server.Server(415): Started @166981ms 2024-11-10T20:28:24,508 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:28:24,604 WARN [Thread-1438 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:28:24,606 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef8d30e0f034d9ed with lease ID 0xcc92c66eeef5aa13: from storage DS-6160c879-2f46-465a-82f3-76e411f22609 node DatanodeRegistration(127.0.0.1:41405, datanodeUuid=8debfdad-9a14-4393-bc0e-9cd36739f6dd, infoPort=35439, infoSecurePort=0, ipcPort=43647, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:24,607 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xef8d30e0f034d9ed with lease ID 0xcc92c66eeef5aa13: from storage DS-0ec36b5d-b380-49be-a594-fee55d3d75d1 node DatanodeRegistration(127.0.0.1:41405, datanodeUuid=8debfdad-9a14-4393-bc0e-9cd36739f6dd, infoPort=35439, infoSecurePort=0, ipcPort=43647, storageInfo=lv=-57;cid=testClusterID;nsid=1778933027;c=1731270485133), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:25,187 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:25,189 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:25,526 INFO [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted 2024-11-10T20:28:25,528 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-11-10T20:28:25,530 ERROR [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504-prefix:734d44df782b,33961,1731270485865 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38393,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:25,530 WARN [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504-prefix:734d44df782b,33961,1731270485865 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38393,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:25,530 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C33961%2C1731270485865:(num 1731270500203) roll requested 2024-11-10T20:28:25,530 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33961%2C1731270485865.1731270505530 2024-11-10T20:28:25,536 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 newFile=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270505530 2024-11-10T20:28:25,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:25,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:25,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:25,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:25,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:25,536 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270505530 2024-11-10T20:28:25,537 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38393,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:25,537 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38393,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:25,537 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 2024-11-10T20:28:25,537 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35439:35439),(127.0.0.1/127.0.0.1:32943:32943)] 2024-11-10T20:28:25,537 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 is not closed yet, will try archiving it next time 2024-11-10T20:28:25,537 WARN [IPC Server handler 3 on default port 37029 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018 2024-11-10T20:28:25,537 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 after 0ms 2024-11-10T20:28:26,188 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:26,189 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:27,188 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:27,190 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:27,538 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33961%2C1731270485865.1731270507538 2024-11-10T20:28:27,544 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270505530 newFile=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 2024-11-10T20:28:27,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:27,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:27,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:27,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:27,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:27,545 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270505530 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 2024-11-10T20:28:27,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35439:35439),(127.0.0.1/127.0.0.1:32943:32943)] 2024-11-10T20:28:27,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 is not closed yet, will try archiving it next time 2024-11-10T20:28:27,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270505530 is not closed yet, will try archiving it next time 2024-11-10T20:28:27,547 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 2024-11-10T20:28:27,547 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 2024-11-10T20:28:27,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741838_1019 (size=1264) 2024-11-10T20:28:27,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741838_1019 (size=1264) 2024-11-10T20:28:27,548 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 after 1ms 2024-11-10T20:28:27,548 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 2024-11-10T20:28:27,548 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 is not closed yet, will try archiving it next time 2024-11-10T20:28:27,557 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731270487149/Put/vlen=218/seqid=0] 2024-11-10T20:28:27,558 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731270496870/Put/vlen=1045/seqid=0] 2024-11-10T20:28:27,558 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270486254 2024-11-10T20:28:27,558 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 2024-11-10T20:28:27,558 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 2024-11-10T20:28:27,558 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 after 0ms 2024-11-10T20:28:27,558 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 2024-11-10T20:28:27,561 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731270500202/Put/vlen=1045/seqid=0] 2024-11-10T20:28:27,562 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731270502215/Put/vlen=1045/seqid=0] 2024-11-10T20:28:27,562 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 2024-11-10T20:28:27,562 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270505530 2024-11-10T20:28:27,562 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270505530 2024-11-10T20:28:27,562 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270505530 after 0ms 2024-11-10T20:28:27,562 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270505530 2024-11-10T20:28:27,565 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731270505529/Put/vlen=1045/seqid=0] 2024-11-10T20:28:27,566 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 2024-11-10T20:28:27,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 2024-11-10T20:28:27,566 WARN [IPC Server handler 1 on default port 37029 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021 2024-11-10T20:28:27,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 after 0ms 2024-11-10T20:28:28,189 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:28,190 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:28,449 WARN [ResponseProcessor for block BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:28,449 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1515544807_22 at /127.0.0.1:47484 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:41405:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47484 dst: /127.0.0.1:41405 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41405 remote=/127.0.0.1:47484]. Total timeout mills is 60000, 59095 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:28,449 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1515544807_22 at /127.0.0.1:35108 [Receiving block BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:41867:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35108 dst: /127.0.0.1:41867 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:28:28,449 WARN [DataStreamer for file /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 block BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41405,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK], DatanodeInfoWithStorage[127.0.0.1:41867,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41405,DS-6160c879-2f46-465a-82f3-76e411f22609,DISK]) is bad. 2024-11-10T20:28:28,450 WARN [DataStreamer for file /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 block BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:28,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741839_1022 (size=85) 2024-11-10T20:28:29,190 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:29,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:29,446 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T20:28:29,538 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270500203 after 4001ms 2024-11-10T20:28:30,190 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:30,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:31,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:31,192 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:31,567 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 after 4001ms 2024-11-10T20:28:31,567 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 2024-11-10T20:28:31,571 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 2024-11-10T20:28:31,572 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f3a13100bb802bf8b0a70eafbc11e37 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-11-10T20:28:31,572 ERROR [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504-prefix:734d44df782b,33961,1731270485865 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:31,572 WARN [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504-prefix:734d44df782b,33961,1731270485865 {}] wal.AbstractFSWAL(2174): append entry failed org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:31,573 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C33961%2C1731270485865:(num 1731270507538) roll requested 2024-11-10T20:28:31,573 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33961%2C1731270485865.1731270511573 2024-11-10T20:28:31,578 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 newFile=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270511573 2024-11-10T20:28:31,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,579 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270511573 2024-11-10T20:28:31,579 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:31,579 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1789915910-172.17.0.2-1731270485133:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:31,579 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 2024-11-10T20:28:31,580 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32943:32943),(127.0.0.1/127.0.0.1:35439:35439)] 2024-11-10T20:28:31,580 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 is not closed yet, will try archiving it next time 2024-11-10T20:28:31,580 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 after 1ms 2024-11-10T20:28:31,580 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.1731270507538 to hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/oldWALs/734d44df782b%2C33961%2C1731270485865.1731270507538 2024-11-10T20:28:31,596 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/default/TestLogRolling-testLogRollOnPipelineRestart/1f3a13100bb802bf8b0a70eafbc11e37/.tmp/info/d031bda29068471d9c0b64d2b76143da is 1080, key is row1002/info:/1731270496870/Put/seqid=0 2024-11-10T20:28:31,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741841_1024 (size=9270) 2024-11-10T20:28:31,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741841_1024 (size=9270) 2024-11-10T20:28:31,602 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/default/TestLogRolling-testLogRollOnPipelineRestart/1f3a13100bb802bf8b0a70eafbc11e37/.tmp/info/d031bda29068471d9c0b64d2b76143da 2024-11-10T20:28:31,609 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/default/TestLogRolling-testLogRollOnPipelineRestart/1f3a13100bb802bf8b0a70eafbc11e37/.tmp/info/d031bda29068471d9c0b64d2b76143da as hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/default/TestLogRolling-testLogRollOnPipelineRestart/1f3a13100bb802bf8b0a70eafbc11e37/info/d031bda29068471d9c0b64d2b76143da 2024-11-10T20:28:31,615 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/default/TestLogRolling-testLogRollOnPipelineRestart/1f3a13100bb802bf8b0a70eafbc11e37/info/d031bda29068471d9c0b64d2b76143da, entries=4, sequenceid=8, filesize=9.1 K 2024-11-10T20:28:31,616 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 1f3a13100bb802bf8b0a70eafbc11e37 in 45ms, sequenceid=8, compaction requested=false 2024-11-10T20:28:31,616 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1f3a13100bb802bf8b0a70eafbc11e37: 2024-11-10T20:28:31,617 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB 2024-11-10T20:28:31,617 ERROR [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504-prefix:734d44df782b,33961,1731270485865.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:31,617 WARN [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504-prefix:734d44df782b,33961,1731270485865.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:31,617 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C33961%2C1731270485865.meta:.meta(num 1731270486639) roll requested 2024-11-10T20:28:31,618 INFO [regionserver/734d44df782b:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33961%2C1731270485865.meta.1731270511617.meta 2024-11-10T20:28:31,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,623 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270511617.meta 2024-11-10T20:28:31,623 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:31,623 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:31,623 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta 2024-11-10T20:28:31,624 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32943:32943),(127.0.0.1/127.0.0.1:35439:35439)] 2024-11-10T20:28:31,624 DEBUG [regionserver/734d44df782b:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta is not closed yet, will try archiving it next time 2024-11-10T20:28:31,624 WARN [IPC Server handler 0 on default port 37029 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1015 2024-11-10T20:28:31,624 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta after 1ms 2024-11-10T20:28:31,639 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/.tmp/info/7eb4a08f3d8c4dc4a7dfa65f55938eee is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37./info:regioninfo/1731270487154/Put/seqid=0 2024-11-10T20:28:31,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741843_1027 (size=7125) 2024-11-10T20:28:31,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741843_1027 (size=7125) 2024-11-10T20:28:31,644 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/.tmp/info/7eb4a08f3d8c4dc4a7dfa65f55938eee 2024-11-10T20:28:31,664 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/.tmp/ns/fa39862c76e04dec9989215602fe3a4e is 43, key is default/ns:d/1731270486682/Put/seqid=0 2024-11-10T20:28:31,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741844_1028 (size=5153) 2024-11-10T20:28:31,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741844_1028 (size=5153) 2024-11-10T20:28:31,669 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/.tmp/ns/fa39862c76e04dec9989215602fe3a4e 2024-11-10T20:28:31,689 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/.tmp/table/a1671c6450ad4a4f8affec1cf3f1b843 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731270487165/Put/seqid=0 2024-11-10T20:28:31,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741845_1029 (size=5438) 2024-11-10T20:28:31,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741845_1029 (size=5438) 2024-11-10T20:28:31,694 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/.tmp/table/a1671c6450ad4a4f8affec1cf3f1b843 2024-11-10T20:28:31,700 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/.tmp/info/7eb4a08f3d8c4dc4a7dfa65f55938eee as hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/info/7eb4a08f3d8c4dc4a7dfa65f55938eee 2024-11-10T20:28:31,706 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/info/7eb4a08f3d8c4dc4a7dfa65f55938eee, entries=10, sequenceid=11, filesize=7.0 K 2024-11-10T20:28:31,707 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/.tmp/ns/fa39862c76e04dec9989215602fe3a4e as hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/ns/fa39862c76e04dec9989215602fe3a4e 2024-11-10T20:28:31,712 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/ns/fa39862c76e04dec9989215602fe3a4e, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T20:28:31,713 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/.tmp/table/a1671c6450ad4a4f8affec1cf3f1b843 as hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/table/a1671c6450ad4a4f8affec1cf3f1b843 2024-11-10T20:28:31,718 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/table/a1671c6450ad4a4f8affec1cf3f1b843, entries=2, sequenceid=11, filesize=5.3 K 2024-11-10T20:28:31,719 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=11, compaction requested=false 2024-11-10T20:28:31,719 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-10T20:28:31,724 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T20:28:31,724 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:28:31,724 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:28:31,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:28:31,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:28:31,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T20:28:31,725 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T20:28:31,725 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=95841690, stopped=false 2024-11-10T20:28:31,725 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=734d44df782b,43943,1731270485818 2024-11-10T20:28:31,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:28:31,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:28:31,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:31,727 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:28:31,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:31,727 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:28:31,727 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:28:31,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:28:31,727 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '734d44df782b,33961,1731270485865' ***** 2024-11-10T20:28:31,727 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T20:28:31,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:28:31,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:28:31,728 INFO [RS:0;734d44df782b:33961 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T20:28:31,728 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T20:28:31,728 INFO [RS:0;734d44df782b:33961 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T20:28:31,728 INFO [RS:0;734d44df782b:33961 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T20:28:31,728 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(3091): Received CLOSE for 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:31,729 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(959): stopping server 734d44df782b,33961,1731270485865 2024-11-10T20:28:31,729 INFO [RS:0;734d44df782b:33961 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:28:31,729 INFO [RS:0;734d44df782b:33961 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;734d44df782b:33961. 2024-11-10T20:28:31,729 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1f3a13100bb802bf8b0a70eafbc11e37, disabling compactions & flushes 2024-11-10T20:28:31,729 DEBUG [RS:0;734d44df782b:33961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:28:31,729 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:31,729 DEBUG [RS:0;734d44df782b:33961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:28:31,729 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:31,729 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. after waiting 0 ms 2024-11-10T20:28:31,729 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:31,729 INFO [RS:0;734d44df782b:33961 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T20:28:31,729 INFO [RS:0;734d44df782b:33961 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T20:28:31,729 INFO [RS:0;734d44df782b:33961 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T20:28:31,729 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T20:28:31,729 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T20:28:31,730 DEBUG [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(1325): Online Regions={1f3a13100bb802bf8b0a70eafbc11e37=TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T20:28:31,730 DEBUG [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f3a13100bb802bf8b0a70eafbc11e37 2024-11-10T20:28:31,730 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:28:31,730 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:28:31,730 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:28:31,730 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:28:31,730 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:28:31,734 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/default/TestLogRolling-testLogRollOnPipelineRestart/1f3a13100bb802bf8b0a70eafbc11e37/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-10T20:28:31,734 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T20:28:31,734 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:31,734 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1f3a13100bb802bf8b0a70eafbc11e37: Waiting for close lock at 1731270511729Running coprocessor pre-close hooks at 1731270511729Disabling compacts and flushes for region at 1731270511729Disabling writes for close at 1731270511729Writing region close event to WAL at 1731270511730 (+1 ms)Running coprocessor post-close hooks at 1731270511734 (+4 ms)Closed at 1731270511734 2024-11-10T20:28:31,735 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731270486796.1f3a13100bb802bf8b0a70eafbc11e37. 2024-11-10T20:28:31,735 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:28:31,735 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:28:31,735 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270511730Running coprocessor pre-close hooks at 1731270511730Disabling compacts and flushes for region at 1731270511730Disabling writes for close at 1731270511730Writing region close event to WAL at 1731270511731 (+1 ms)Running coprocessor post-close hooks at 1731270511735 (+4 ms)Closed at 1731270511735 2024-11-10T20:28:31,735 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T20:28:31,930 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(976): stopping server 734d44df782b,33961,1731270485865; all regions closed. 2024-11-10T20:28:31,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:31,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741842_1025 (size=825) 2024-11-10T20:28:31,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741842_1025 (size=825) 2024-11-10T20:28:32,121 INFO [regionserver/734d44df782b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:28:32,122 INFO [regionserver/734d44df782b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T20:28:32,122 INFO [regionserver/734d44df782b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T20:28:32,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:32,192 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:32,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:28:32,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T20:28:32,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T20:28:33,192 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:33,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:34,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:34,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:35,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:35,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:35,607 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1015: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T20:28:35,625 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta after 4002ms 2024-11-10T20:28:35,625 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/WALs/734d44df782b,33961,1731270485865/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta to hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/oldWALs/734d44df782b%2C33961%2C1731270485865.meta.1731270486639.meta 2024-11-10T20:28:35,628 DEBUG [RS:0;734d44df782b:33961 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/oldWALs 2024-11-10T20:28:35,628 INFO [RS:0;734d44df782b:33961 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C33961%2C1731270485865.meta:.meta(num 1731270511617) 2024-11-10T20:28:35,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741840_1023 (size=1162) 2024-11-10T20:28:35,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741840_1023 (size=1162) 2024-11-10T20:28:35,636 DEBUG [RS:0;734d44df782b:33961 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/oldWALs 2024-11-10T20:28:35,636 INFO [RS:0;734d44df782b:33961 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C33961%2C1731270485865:(num 1731270511573) 2024-11-10T20:28:35,636 DEBUG [RS:0;734d44df782b:33961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:28:35,636 INFO [RS:0;734d44df782b:33961 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:28:35,636 INFO [RS:0;734d44df782b:33961 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:28:35,636 INFO [RS:0;734d44df782b:33961 {}] hbase.ChoreService(370): Chore service for: regionserver/734d44df782b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T20:28:35,636 INFO [RS:0;734d44df782b:33961 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:28:35,636 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:28:35,636 INFO [RS:0;734d44df782b:33961 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33961 2024-11-10T20:28:35,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/734d44df782b,33961,1731270485865 2024-11-10T20:28:35,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:28:35,638 INFO [RS:0;734d44df782b:33961 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:28:35,640 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [734d44df782b,33961,1731270485865] 2024-11-10T20:28:35,641 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/734d44df782b,33961,1731270485865 already deleted, retry=false 2024-11-10T20:28:35,641 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 734d44df782b,33961,1731270485865 expired; onlineServers=0 2024-11-10T20:28:35,641 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '734d44df782b,43943,1731270485818' ***** 2024-11-10T20:28:35,641 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T20:28:35,641 INFO [M:0;734d44df782b:43943 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:28:35,641 INFO [M:0;734d44df782b:43943 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:28:35,642 DEBUG [M:0;734d44df782b:43943 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T20:28:35,642 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T20:28:35,642 DEBUG [M:0;734d44df782b:43943 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T20:28:35,642 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270486033 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270486033,5,FailOnTimeoutGroup] 2024-11-10T20:28:35,642 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270486033 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270486033,5,FailOnTimeoutGroup] 2024-11-10T20:28:35,642 INFO [M:0;734d44df782b:43943 {}] hbase.ChoreService(370): Chore service for: master/734d44df782b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T20:28:35,642 INFO [M:0;734d44df782b:43943 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:28:35,642 DEBUG [M:0;734d44df782b:43943 {}] master.HMaster(1795): Stopping service threads 2024-11-10T20:28:35,642 INFO [M:0;734d44df782b:43943 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T20:28:35,642 INFO [M:0;734d44df782b:43943 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:28:35,643 INFO [M:0;734d44df782b:43943 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T20:28:35,643 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T20:28:35,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T20:28:35,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:35,643 DEBUG [M:0;734d44df782b:43943 {}] zookeeper.ZKUtil(347): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T20:28:35,643 WARN [M:0;734d44df782b:43943 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T20:28:35,644 INFO [M:0;734d44df782b:43943 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/.lastflushedseqids 2024-11-10T20:28:35,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741846_1030 (size=130) 2024-11-10T20:28:35,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741846_1030 (size=130) 2024-11-10T20:28:35,652 INFO [M:0;734d44df782b:43943 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T20:28:35,653 INFO [M:0;734d44df782b:43943 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T20:28:35,653 DEBUG [M:0;734d44df782b:43943 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:28:35,653 INFO [M:0;734d44df782b:43943 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:35,653 DEBUG [M:0;734d44df782b:43943 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:35,653 DEBUG [M:0;734d44df782b:43943 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:28:35,653 DEBUG [M:0;734d44df782b:43943 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:35,653 INFO [M:0;734d44df782b:43943 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB 2024-11-10T20:28:35,654 ERROR [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData-prefix:734d44df782b,43943,1731270485818 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:35,654 WARN [FSHLog-0-hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData-prefix:734d44df782b,43943,1731270485818 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:35,654 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 734d44df782b%2C43943%2C1731270485818:(num 1731270485951) roll requested 2024-11-10T20:28:35,654 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C43943%2C1731270485818.1731270515654 2024-11-10T20:28:35,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,660 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270485951 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270515654 2024-11-10T20:28:35,661 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:35,661 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41653,DS-6e950ebc-3b6c-4011-99e1-1ed0aea7e727,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T20:28:35,661 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270485951 2024-11-10T20:28:35,661 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35439:35439),(127.0.0.1/127.0.0.1:32943:32943)] 2024-11-10T20:28:35,661 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270485951 is not closed yet, will try archiving it next time 2024-11-10T20:28:35,661 WARN [IPC Server handler 2 on default port 37029 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270485951 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013 2024-11-10T20:28:35,662 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270485951 after 1ms 2024-11-10T20:28:35,677 DEBUG [M:0;734d44df782b:43943 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0a7b5866bde642ca9af7ef08083eedad is 82, key is hbase:meta,,1/info:regioninfo/1731270486665/Put/seqid=0 2024-11-10T20:28:35,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741848_1033 (size=5672) 2024-11-10T20:28:35,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741848_1033 (size=5672) 2024-11-10T20:28:35,683 INFO [M:0;734d44df782b:43943 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0a7b5866bde642ca9af7ef08083eedad 2024-11-10T20:28:35,702 DEBUG [M:0;734d44df782b:43943 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b8d19c72d9e749e989122c7df42f8289 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731270487169/Put/seqid=0 2024-11-10T20:28:35,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741849_1034 (size=6118) 2024-11-10T20:28:35,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741849_1034 (size=6118) 2024-11-10T20:28:35,708 INFO [M:0;734d44df782b:43943 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b8d19c72d9e749e989122c7df42f8289 2024-11-10T20:28:35,728 DEBUG [M:0;734d44df782b:43943 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74f5bd092d244dbead15103de95c4ae2 is 69, key is 734d44df782b,33961,1731270485865/rs:state/1731270486103/Put/seqid=0 2024-11-10T20:28:35,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741850_1035 (size=5156) 2024-11-10T20:28:35,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741850_1035 (size=5156) 2024-11-10T20:28:35,733 INFO [M:0;734d44df782b:43943 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74f5bd092d244dbead15103de95c4ae2 2024-11-10T20:28:35,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:35,740 INFO [RS:0;734d44df782b:33961 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:28:35,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33961-0x10193de9c930001, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:35,740 INFO [RS:0;734d44df782b:33961 {}] regionserver.HRegionServer(1031): Exiting; stopping=734d44df782b,33961,1731270485865; zookeeper connection closed. 2024-11-10T20:28:35,740 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a6bb9ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a6bb9ae 2024-11-10T20:28:35,741 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T20:28:35,753 DEBUG [M:0;734d44df782b:43943 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/91f0a5cd1ff6468aaee5a830324d1676 is 52, key is load_balancer_on/state:d/1731270486792/Put/seqid=0 2024-11-10T20:28:35,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741851_1036 (size=5056) 2024-11-10T20:28:35,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741851_1036 (size=5056) 2024-11-10T20:28:35,758 INFO [M:0;734d44df782b:43943 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/91f0a5cd1ff6468aaee5a830324d1676 2024-11-10T20:28:35,763 DEBUG [M:0;734d44df782b:43943 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0a7b5866bde642ca9af7ef08083eedad as hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0a7b5866bde642ca9af7ef08083eedad 2024-11-10T20:28:35,768 INFO [M:0;734d44df782b:43943 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0a7b5866bde642ca9af7ef08083eedad, entries=8, sequenceid=56, filesize=5.5 K 2024-11-10T20:28:35,769 DEBUG [M:0;734d44df782b:43943 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b8d19c72d9e749e989122c7df42f8289 as hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b8d19c72d9e749e989122c7df42f8289 2024-11-10T20:28:35,773 INFO [M:0;734d44df782b:43943 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b8d19c72d9e749e989122c7df42f8289, entries=6, sequenceid=56, filesize=6.0 K 2024-11-10T20:28:35,774 DEBUG [M:0;734d44df782b:43943 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74f5bd092d244dbead15103de95c4ae2 as hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74f5bd092d244dbead15103de95c4ae2 2024-11-10T20:28:35,779 INFO [M:0;734d44df782b:43943 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74f5bd092d244dbead15103de95c4ae2, entries=1, sequenceid=56, filesize=5.0 K 2024-11-10T20:28:35,780 DEBUG [M:0;734d44df782b:43943 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/91f0a5cd1ff6468aaee5a830324d1676 as hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/91f0a5cd1ff6468aaee5a830324d1676 2024-11-10T20:28:35,784 INFO [M:0;734d44df782b:43943 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/91f0a5cd1ff6468aaee5a830324d1676, entries=1, sequenceid=56, filesize=4.9 K 2024-11-10T20:28:35,786 INFO [M:0;734d44df782b:43943 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 132ms, sequenceid=56, compaction requested=false 2024-11-10T20:28:35,787 INFO [M:0;734d44df782b:43943 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:35,787 DEBUG [M:0;734d44df782b:43943 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270515653Disabling compacts and flushes for region at 1731270515653Disabling writes for close at 1731270515653Obtaining lock to block concurrent updates at 1731270515653Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731270515653Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731270515654 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731270515662 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731270515662Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731270515677 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731270515677Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731270515688 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731270515702 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731270515702Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731270515713 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731270515727 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731270515727Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731270515738 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731270515752 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731270515752Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@863df2f: reopening flushed file at 1731270515762 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a82c3f5: reopening flushed file at 1731270515768 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ece308a: reopening flushed file at 1731270515773 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55d5eea4: reopening flushed file at 1731270515779 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 132ms, sequenceid=56, compaction requested=false at 1731270515786 (+7 ms)Writing region close event to WAL at 1731270515787 (+1 ms)Closed at 1731270515787 2024-11-10T20:28:35,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:28:35,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741847_1031 (size=757) 2024-11-10T20:28:35,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41405 is added to blk_1073741847_1031 (size=757) 2024-11-10T20:28:35,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T20:28:36,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:36,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:36,735 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,735 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,749 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,749 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,749 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,750 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,750 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,750 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,753 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,753 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,753 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,754 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,758 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:36,758 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:37,195 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:37,260 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T20:28:37,262 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,262 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,262 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,262 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,277 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,277 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,278 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,278 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,278 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,278 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,282 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,282 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,282 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:37,284 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:38,195 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:38,195 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:38,609 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T20:28:39,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:39,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:39,662 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270485951 after 4001ms 2024-11-10T20:28:39,663 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/WALs/734d44df782b,43943,1731270485818/734d44df782b%2C43943%2C1731270485818.1731270485951 to hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/oldWALs/734d44df782b%2C43943%2C1731270485818.1731270485951 2024-11-10T20:28:39,666 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/MasterData/oldWALs/734d44df782b%2C43943%2C1731270485818.1731270485951 to hdfs://localhost:37029/user/jenkins/test-data/205f4b98-273d-afdc-49a7-844ee1a20504/oldWALs/734d44df782b%2C43943%2C1731270485818.1731270485951$masterlocalwal$ 2024-11-10T20:28:39,666 INFO [M:0;734d44df782b:43943 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T20:28:39,666 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:28:39,666 INFO [M:0;734d44df782b:43943 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43943 2024-11-10T20:28:39,667 INFO [M:0;734d44df782b:43943 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:28:39,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:39,769 INFO [M:0;734d44df782b:43943 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:28:39,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43943-0x10193de9c930000, quorum=127.0.0.1:59131, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:28:39,771 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21404da7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:39,772 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e8914c8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:39,772 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:39,772 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6baabd83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:39,772 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3146549d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:39,773 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:28:39,773 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:28:39,773 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1789915910-172.17.0.2-1731270485133 (Datanode Uuid 8debfdad-9a14-4393-bc0e-9cd36739f6dd) service to localhost/127.0.0.1:37029 2024-11-10T20:28:39,773 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:28:39,774 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data3/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:39,774 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data4/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:39,774 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:28:39,776 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1dcfabbb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:39,777 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6614137b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:39,777 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:39,777 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62802d30{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:39,777 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@717499fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:39,778 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:28:39,778 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:28:39,778 WARN [BP-1789915910-172.17.0.2-1731270485133 heartbeating to localhost/127.0.0.1:37029 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1789915910-172.17.0.2-1731270485133 (Datanode Uuid 293800f1-564b-45e1-a843-e68e7f40dc6e) service to localhost/127.0.0.1:37029 2024-11-10T20:28:39,778 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:28:39,779 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data1/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:39,779 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/cluster_0789af39-283b-9971-c810-c99f95335c8b/data/data2/current/BP-1789915910-172.17.0.2-1731270485133 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:28:39,779 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:28:39,784 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65e5bfc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:28:39,785 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c23e5e9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:28:39,785 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:28:39,785 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dbadaea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:28:39,785 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a3130af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir/,STOPPED} 2024-11-10T20:28:39,791 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T20:28:39,808 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T20:28:39,817 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 154) Potentially hanging thread: nioEventLoopGroup-33-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37029 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:37029 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37029 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37029 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37029 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37029 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37029 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37029 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=18 (was 25), ProcessCount=11 (was 11), AvailableMemoryMB=7331 (was 7429) 2024-11-10T20:28:39,825 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=18, ProcessCount=11, AvailableMemoryMB=7331 2024-11-10T20:28:39,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T20:28:39,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.log.dir so I do NOT create it in target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98 2024-11-10T20:28:39,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8de1ea3e-5cb4-313c-b714-331d33f64c03/hadoop.tmp.dir so I do NOT create it in target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98 2024-11-10T20:28:39,825 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00, deleteOnExit=true 2024-11-10T20:28:39,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T20:28:39,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/test.cache.data in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.log.dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T20:28:39,826 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:28:39,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:28:39,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T20:28:39,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/nfs.dump.dir in system properties and HBase conf 2024-11-10T20:28:39,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/java.io.tmpdir in system properties and HBase conf 2024-11-10T20:28:39,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:28:39,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T20:28:39,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T20:28:39,840 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:28:39,906 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:39,912 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:39,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:39,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:39,916 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:28:39,917 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:39,917 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62ee2d8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:39,917 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13b3ff43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:40,032 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@521d1c5a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/java.io.tmpdir/jetty-localhost-45371-hadoop-hdfs-3_4_1-tests_jar-_-any-443619560781419349/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:28:40,033 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@62e6de45{HTTP/1.1, (http/1.1)}{localhost:45371} 2024-11-10T20:28:40,033 INFO [Time-limited test {}] server.Server(415): Started @182507ms 2024-11-10T20:28:40,045 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:28:40,103 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:40,106 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:40,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:40,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:40,107 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:28:40,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4eb40860{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:40,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@461cdaba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:40,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:40,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:40,222 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@68744dfe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/java.io.tmpdir/jetty-localhost-33827-hadoop-hdfs-3_4_1-tests_jar-_-any-2785410859629863143/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:40,223 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6eebb3dc{HTTP/1.1, (http/1.1)}{localhost:33827} 2024-11-10T20:28:40,223 INFO [Time-limited test {}] server.Server(415): Started @182697ms 2024-11-10T20:28:40,224 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:28:40,258 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:28:40,261 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:28:40,262 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:28:40,262 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:28:40,262 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:28:40,262 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c020752{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:28:40,263 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47d3f616{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:28:40,310 WARN [Thread-1632 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/data/data1/current/BP-484077747-172.17.0.2-1731270519856/current, will proceed with Du for space computation calculation, 2024-11-10T20:28:40,310 WARN [Thread-1633 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/data/data2/current/BP-484077747-172.17.0.2-1731270519856/current, will proceed with Du for space computation calculation, 2024-11-10T20:28:40,326 WARN [Thread-1611 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:28:40,329 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9468a7ca08c1a67f with lease ID 0xfb8e0f92db56be97: Processing first storage report for DS-ea29cf2e-2068-4fb2-9780-6e650f564f1d from datanode DatanodeRegistration(127.0.0.1:45271, datanodeUuid=95c6cd7c-cd6c-4b11-adec-eadbb917afa4, infoPort=36873, infoSecurePort=0, ipcPort=41957, storageInfo=lv=-57;cid=testClusterID;nsid=1150967267;c=1731270519856) 2024-11-10T20:28:40,329 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9468a7ca08c1a67f with lease ID 0xfb8e0f92db56be97: from storage DS-ea29cf2e-2068-4fb2-9780-6e650f564f1d node DatanodeRegistration(127.0.0.1:45271, datanodeUuid=95c6cd7c-cd6c-4b11-adec-eadbb917afa4, infoPort=36873, infoSecurePort=0, ipcPort=41957, storageInfo=lv=-57;cid=testClusterID;nsid=1150967267;c=1731270519856), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:40,329 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9468a7ca08c1a67f with lease ID 0xfb8e0f92db56be97: Processing first storage report for DS-cd9cae9f-25a8-4a55-8937-0e2ecff24b8a from datanode DatanodeRegistration(127.0.0.1:45271, datanodeUuid=95c6cd7c-cd6c-4b11-adec-eadbb917afa4, infoPort=36873, infoSecurePort=0, ipcPort=41957, storageInfo=lv=-57;cid=testClusterID;nsid=1150967267;c=1731270519856) 2024-11-10T20:28:40,329 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9468a7ca08c1a67f with lease ID 0xfb8e0f92db56be97: from storage DS-cd9cae9f-25a8-4a55-8937-0e2ecff24b8a node DatanodeRegistration(127.0.0.1:45271, datanodeUuid=95c6cd7c-cd6c-4b11-adec-eadbb917afa4, infoPort=36873, infoSecurePort=0, ipcPort=41957, storageInfo=lv=-57;cid=testClusterID;nsid=1150967267;c=1731270519856), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:40,378 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@374dfdaf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/java.io.tmpdir/jetty-localhost-41673-hadoop-hdfs-3_4_1-tests_jar-_-any-16483908342570838736/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:28:40,378 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@46a495b0{HTTP/1.1, (http/1.1)}{localhost:41673} 2024-11-10T20:28:40,378 INFO [Time-limited test {}] server.Server(415): Started @182853ms 2024-11-10T20:28:40,380 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:28:40,474 WARN [Thread-1658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/data/data3/current/BP-484077747-172.17.0.2-1731270519856/current, will proceed with Du for space computation calculation, 2024-11-10T20:28:40,474 WARN [Thread-1659 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/data/data4/current/BP-484077747-172.17.0.2-1731270519856/current, will proceed with Du for space computation calculation, 2024-11-10T20:28:40,491 WARN [Thread-1647 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:28:40,493 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x964fa53ffa05ca1b with lease ID 0xfb8e0f92db56be98: Processing first storage report for DS-1d570460-d104-47e8-af0b-b930d6d2c3ee from datanode DatanodeRegistration(127.0.0.1:42043, datanodeUuid=ba12c7d2-f7f6-42c9-9e36-65b58ff0ff90, infoPort=39575, infoSecurePort=0, ipcPort=41927, storageInfo=lv=-57;cid=testClusterID;nsid=1150967267;c=1731270519856) 2024-11-10T20:28:40,493 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x964fa53ffa05ca1b with lease ID 0xfb8e0f92db56be98: from storage DS-1d570460-d104-47e8-af0b-b930d6d2c3ee node DatanodeRegistration(127.0.0.1:42043, datanodeUuid=ba12c7d2-f7f6-42c9-9e36-65b58ff0ff90, infoPort=39575, infoSecurePort=0, ipcPort=41927, storageInfo=lv=-57;cid=testClusterID;nsid=1150967267;c=1731270519856), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:40,493 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x964fa53ffa05ca1b with lease ID 0xfb8e0f92db56be98: Processing first storage report for DS-c5b23fc5-deb3-4d28-b74c-5a339ac7ffc5 from datanode DatanodeRegistration(127.0.0.1:42043, datanodeUuid=ba12c7d2-f7f6-42c9-9e36-65b58ff0ff90, infoPort=39575, infoSecurePort=0, ipcPort=41927, storageInfo=lv=-57;cid=testClusterID;nsid=1150967267;c=1731270519856) 2024-11-10T20:28:40,493 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x964fa53ffa05ca1b with lease ID 0xfb8e0f92db56be98: from storage DS-c5b23fc5-deb3-4d28-b74c-5a339ac7ffc5 node DatanodeRegistration(127.0.0.1:42043, datanodeUuid=ba12c7d2-f7f6-42c9-9e36-65b58ff0ff90, infoPort=39575, infoSecurePort=0, ipcPort=41927, storageInfo=lv=-57;cid=testClusterID;nsid=1150967267;c=1731270519856), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:28:40,502 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98 2024-11-10T20:28:40,505 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/zookeeper_0, clientPort=53833, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T20:28:40,506 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53833 2024-11-10T20:28:40,506 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:40,507 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:40,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:28:40,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:28:40,517 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2 with version=8 2024-11-10T20:28:40,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/hbase-staging 2024-11-10T20:28:40,519 INFO [Time-limited test {}] client.ConnectionUtils(128): master/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:28:40,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:40,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:40,520 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:28:40,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:40,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:28:40,520 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T20:28:40,520 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:28:40,521 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34959 2024-11-10T20:28:40,523 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34959 connecting to ZooKeeper ensemble=127.0.0.1:53833 2024-11-10T20:28:40,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349590x0, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:28:40,532 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34959-0x10193df24200000 connected 2024-11-10T20:28:40,546 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:40,547 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:40,549 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:28:40,549 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2, hbase.cluster.distributed=false 2024-11-10T20:28:40,551 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:28:40,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34959 2024-11-10T20:28:40,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34959 2024-11-10T20:28:40,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34959 2024-11-10T20:28:40,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34959 2024-11-10T20:28:40,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34959 2024-11-10T20:28:40,567 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:28:40,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:40,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:40,567 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:28:40,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:28:40,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:28:40,567 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T20:28:40,567 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:28:40,568 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44249 2024-11-10T20:28:40,569 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44249 connecting to ZooKeeper ensemble=127.0.0.1:53833 2024-11-10T20:28:40,570 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:40,571 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:40,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442490x0, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:28:40,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442490x0, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:28:40,576 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44249-0x10193df24200001 connected 2024-11-10T20:28:40,576 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T20:28:40,577 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T20:28:40,577 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T20:28:40,578 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:28:40,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44249 2024-11-10T20:28:40,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44249 2024-11-10T20:28:40,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44249 2024-11-10T20:28:40,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44249 2024-11-10T20:28:40,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44249 2024-11-10T20:28:40,591 DEBUG [M:0;734d44df782b:34959 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;734d44df782b:34959 2024-11-10T20:28:40,591 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/734d44df782b,34959,1731270520519 2024-11-10T20:28:40,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:28:40,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:28:40,594 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/734d44df782b,34959,1731270520519 2024-11-10T20:28:40,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T20:28:40,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,596 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T20:28:40,596 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/734d44df782b,34959,1731270520519 from backup master directory 2024-11-10T20:28:40,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/734d44df782b,34959,1731270520519 2024-11-10T20:28:40,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:28:40,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:28:40,599 WARN [master/734d44df782b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:28:40,599 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=734d44df782b,34959,1731270520519 2024-11-10T20:28:40,603 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/hbase.id] with ID: 8928a194-422a-4e34-baef-d1b363868780 2024-11-10T20:28:40,603 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/.tmp/hbase.id 2024-11-10T20:28:40,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:28:40,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:28:40,610 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/.tmp/hbase.id]:[hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/hbase.id] 2024-11-10T20:28:40,620 INFO [master/734d44df782b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:40,620 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T20:28:40,622 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-10T20:28:40,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:28:40,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:28:40,632 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:28:40,632 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T20:28:40,633 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:28:40,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:28:40,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:28:40,641 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store 2024-11-10T20:28:40,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:28:40,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:28:40,647 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:40,647 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:28:40,647 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:40,647 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:40,648 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:28:40,648 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:40,648 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:28:40,648 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270520647Disabling compacts and flushes for region at 1731270520647Disabling writes for close at 1731270520648 (+1 ms)Writing region close event to WAL at 1731270520648Closed at 1731270520648 2024-11-10T20:28:40,648 WARN [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/.initializing 2024-11-10T20:28:40,649 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/WALs/734d44df782b,34959,1731270520519 2024-11-10T20:28:40,651 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C34959%2C1731270520519, suffix=, logDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/WALs/734d44df782b,34959,1731270520519, archiveDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/oldWALs, maxLogs=10 2024-11-10T20:28:40,651 INFO [master/734d44df782b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C34959%2C1731270520519.1731270520651 2024-11-10T20:28:40,656 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/WALs/734d44df782b,34959,1731270520519/734d44df782b%2C34959%2C1731270520519.1731270520651 2024-11-10T20:28:40,658 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36873:36873),(127.0.0.1/127.0.0.1:39575:39575)] 2024-11-10T20:28:40,658 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:28:40,658 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:40,659 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,659 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T20:28:40,661 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:40,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:40,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T20:28:40,663 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:40,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:28:40,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,664 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T20:28:40,664 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:40,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:28:40,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T20:28:40,666 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:40,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:28:40,666 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,667 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,667 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,669 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,669 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,670 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T20:28:40,671 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:28:40,673 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:28:40,674 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=827252, jitterRate=0.05190563201904297}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T20:28:40,675 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731270520659Initializing all the Stores at 1731270520659Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270520659Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270520660 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270520660Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270520660Cleaning up temporary data from old regions at 1731270520669 (+9 ms)Region opened successfully at 1731270520674 (+5 ms) 2024-11-10T20:28:40,675 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T20:28:40,678 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ea6c30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:28:40,679 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T20:28:40,679 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T20:28:40,679 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T20:28:40,679 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T20:28:40,679 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T20:28:40,680 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T20:28:40,680 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T20:28:40,682 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T20:28:40,682 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T20:28:40,684 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T20:28:40,685 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T20:28:40,685 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T20:28:40,687 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T20:28:40,688 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T20:28:40,688 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T20:28:40,691 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T20:28:40,692 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T20:28:40,696 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T20:28:40,698 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T20:28:40,699 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T20:28:40,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:28:40,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:28:40,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,701 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=734d44df782b,34959,1731270520519, sessionid=0x10193df24200000, setting cluster-up flag (Was=false) 2024-11-10T20:28:40,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,710 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T20:28:40,711 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,34959,1731270520519 2024-11-10T20:28:40,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:40,722 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T20:28:40,723 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,34959,1731270520519 2024-11-10T20:28:40,724 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T20:28:40,726 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T20:28:40,726 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T20:28:40,726 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T20:28:40,726 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 734d44df782b,34959,1731270520519 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T20:28:40,727 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:28:40,727 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:28:40,727 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:28:40,727 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:28:40,727 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/734d44df782b:0, corePoolSize=10, maxPoolSize=10 2024-11-10T20:28:40,727 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,727 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:28:40,727 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,728 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731270550728 2024-11-10T20:28:40,728 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T20:28:40,728 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T20:28:40,728 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T20:28:40,728 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T20:28:40,728 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T20:28:40,729 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T20:28:40,729 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,729 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T20:28:40,729 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T20:28:40,729 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T20:28:40,729 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:28:40,729 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T20:28:40,729 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T20:28:40,729 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T20:28:40,730 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270520729,5,FailOnTimeoutGroup] 2024-11-10T20:28:40,730 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270520730,5,FailOnTimeoutGroup] 2024-11-10T20:28:40,730 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,730 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T20:28:40,730 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,730 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,730 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:40,730 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T20:28:40,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:28:40,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:28:40,737 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T20:28:40,738 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2 2024-11-10T20:28:40,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:28:40,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:28:40,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:40,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:28:40,746 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:28:40,746 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:40,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:40,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:28:40,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:28:40,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:40,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:40,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:28:40,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:28:40,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:40,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:40,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:28:40,751 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:28:40,751 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:40,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:40,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:28:40,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740 2024-11-10T20:28:40,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740 2024-11-10T20:28:40,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:28:40,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:28:40,754 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:28:40,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:28:40,757 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:28:40,757 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=834319, jitterRate=0.060891762375831604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:28:40,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731270520744Initializing all the Stores at 1731270520745 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270520745Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270520745Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270520745Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270520745Cleaning up temporary data from old regions at 1731270520754 (+9 ms)Region opened successfully at 1731270520758 (+4 ms) 2024-11-10T20:28:40,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:28:40,758 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:28:40,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:28:40,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:28:40,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:28:40,758 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:28:40,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270520758Disabling compacts and flushes for region at 1731270520758Disabling writes for close at 1731270520758Writing region close event to WAL at 1731270520758Closed at 1731270520758 2024-11-10T20:28:40,760 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:28:40,760 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T20:28:40,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T20:28:40,761 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:28:40,762 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T20:28:40,781 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(746): ClusterId : 8928a194-422a-4e34-baef-d1b363868780 2024-11-10T20:28:40,781 DEBUG [RS:0;734d44df782b:44249 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T20:28:40,783 DEBUG [RS:0;734d44df782b:44249 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T20:28:40,783 DEBUG [RS:0;734d44df782b:44249 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T20:28:40,785 DEBUG [RS:0;734d44df782b:44249 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T20:28:40,785 DEBUG [RS:0;734d44df782b:44249 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@614aad9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:28:40,797 DEBUG [RS:0;734d44df782b:44249 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;734d44df782b:44249 2024-11-10T20:28:40,797 INFO [RS:0;734d44df782b:44249 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T20:28:40,797 INFO [RS:0;734d44df782b:44249 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T20:28:40,797 DEBUG [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T20:28:40,798 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(2659): reportForDuty to master=734d44df782b,34959,1731270520519 with port=44249, startcode=1731270520567 2024-11-10T20:28:40,798 DEBUG [RS:0;734d44df782b:44249 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T20:28:40,800 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50413, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T20:28:40,801 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34959 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 734d44df782b,44249,1731270520567 2024-11-10T20:28:40,801 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34959 {}] master.ServerManager(517): Registering regionserver=734d44df782b,44249,1731270520567 2024-11-10T20:28:40,803 DEBUG [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2 2024-11-10T20:28:40,803 DEBUG [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35011 2024-11-10T20:28:40,803 DEBUG [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T20:28:40,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:28:40,805 DEBUG [RS:0;734d44df782b:44249 {}] zookeeper.ZKUtil(111): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/734d44df782b,44249,1731270520567 2024-11-10T20:28:40,805 WARN [RS:0;734d44df782b:44249 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:28:40,805 INFO [RS:0;734d44df782b:44249 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:28:40,805 DEBUG [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567 2024-11-10T20:28:40,805 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [734d44df782b,44249,1731270520567] 2024-11-10T20:28:40,808 INFO [RS:0;734d44df782b:44249 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T20:28:40,809 INFO [RS:0;734d44df782b:44249 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T20:28:40,810 INFO [RS:0;734d44df782b:44249 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T20:28:40,810 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,810 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T20:28:40,811 INFO [RS:0;734d44df782b:44249 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T20:28:40,811 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:28:40,811 DEBUG [RS:0;734d44df782b:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:28:40,812 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,812 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,812 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,812 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,812 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,812 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,44249,1731270520567-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:28:40,827 INFO [RS:0;734d44df782b:44249 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T20:28:40,827 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,44249,1731270520567-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,827 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,827 INFO [RS:0;734d44df782b:44249 {}] regionserver.Replication(171): 734d44df782b,44249,1731270520567 started 2024-11-10T20:28:40,841 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:40,841 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(1482): Serving as 734d44df782b,44249,1731270520567, RpcServer on 734d44df782b/172.17.0.2:44249, sessionid=0x10193df24200001 2024-11-10T20:28:40,841 DEBUG [RS:0;734d44df782b:44249 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T20:28:40,841 DEBUG [RS:0;734d44df782b:44249 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 734d44df782b,44249,1731270520567 2024-11-10T20:28:40,841 DEBUG [RS:0;734d44df782b:44249 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,44249,1731270520567' 2024-11-10T20:28:40,841 DEBUG [RS:0;734d44df782b:44249 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T20:28:40,842 DEBUG [RS:0;734d44df782b:44249 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T20:28:40,842 DEBUG [RS:0;734d44df782b:44249 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T20:28:40,842 DEBUG [RS:0;734d44df782b:44249 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T20:28:40,842 DEBUG [RS:0;734d44df782b:44249 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 734d44df782b,44249,1731270520567 2024-11-10T20:28:40,842 DEBUG [RS:0;734d44df782b:44249 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,44249,1731270520567' 2024-11-10T20:28:40,842 DEBUG [RS:0;734d44df782b:44249 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T20:28:40,843 DEBUG [RS:0;734d44df782b:44249 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T20:28:40,843 DEBUG [RS:0;734d44df782b:44249 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T20:28:40,843 INFO [RS:0;734d44df782b:44249 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T20:28:40,843 INFO [RS:0;734d44df782b:44249 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T20:28:40,912 WARN [734d44df782b:34959 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T20:28:40,945 INFO [RS:0;734d44df782b:44249 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C44249%2C1731270520567, suffix=, logDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567, archiveDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/oldWALs, maxLogs=32 2024-11-10T20:28:40,946 INFO [RS:0;734d44df782b:44249 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C44249%2C1731270520567.1731270520945 2024-11-10T20:28:40,951 INFO [RS:0;734d44df782b:44249 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270520945 2024-11-10T20:28:40,952 DEBUG [RS:0;734d44df782b:44249 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39575:39575),(127.0.0.1/127.0.0.1:36873:36873)] 2024-11-10T20:28:41,162 DEBUG [734d44df782b:34959 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T20:28:41,163 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=734d44df782b,44249,1731270520567 2024-11-10T20:28:41,164 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,44249,1731270520567, state=OPENING 2024-11-10T20:28:41,166 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T20:28:41,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:41,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:28:41,168 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:28:41,168 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:28:41,168 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:28:41,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,44249,1731270520567}] 2024-11-10T20:28:41,197 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:41,197 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:41,321 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T20:28:41,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55229, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T20:28:41,327 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T20:28:41,327 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:28:41,329 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C44249%2C1731270520567.meta, suffix=.meta, logDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567, archiveDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/oldWALs, maxLogs=32 2024-11-10T20:28:41,329 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C44249%2C1731270520567.meta.1731270521329.meta 2024-11-10T20:28:41,334 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.meta.1731270521329.meta 2024-11-10T20:28:41,335 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39575:39575),(127.0.0.1/127.0.0.1:36873:36873)] 2024-11-10T20:28:41,336 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:28:41,336 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T20:28:41,336 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T20:28:41,336 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T20:28:41,336 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T20:28:41,336 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:41,336 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T20:28:41,337 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T20:28:41,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:28:41,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:28:41,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:41,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:41,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:28:41,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:28:41,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:41,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:41,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:28:41,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:28:41,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:41,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:41,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:28:41,342 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:28:41,342 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:41,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:28:41,342 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:28:41,343 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740 2024-11-10T20:28:41,344 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740 2024-11-10T20:28:41,345 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:28:41,345 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:28:41,346 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:28:41,347 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:28:41,347 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=838550, jitterRate=0.06627222895622253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:28:41,347 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T20:28:41,348 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731270521337Writing region info on filesystem at 1731270521337Initializing all the Stores at 1731270521337Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270521337Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270521338 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270521338Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270521338Cleaning up temporary data from old regions at 1731270521345 (+7 ms)Running coprocessor post-open hooks at 1731270521347 (+2 ms)Region opened successfully at 1731270521348 (+1 ms) 2024-11-10T20:28:41,349 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731270521321 2024-11-10T20:28:41,352 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T20:28:41,352 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T20:28:41,353 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,44249,1731270520567 2024-11-10T20:28:41,354 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,44249,1731270520567, state=OPEN 2024-11-10T20:28:41,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:28:41,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:28:41,358 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=734d44df782b,44249,1731270520567 2024-11-10T20:28:41,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:28:41,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:28:41,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T20:28:41,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,44249,1731270520567 in 190 msec 2024-11-10T20:28:41,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T20:28:41,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 601 msec 2024-11-10T20:28:41,365 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:28:41,365 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T20:28:41,366 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:28:41,366 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,44249,1731270520567, seqNum=-1] 2024-11-10T20:28:41,366 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:28:41,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:28:41,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 648 msec 2024-11-10T20:28:41,374 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731270521374, completionTime=-1 2024-11-10T20:28:41,374 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T20:28:41,374 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T20:28:41,376 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T20:28:41,376 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731270581376 2024-11-10T20:28:41,376 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731270641376 2024-11-10T20:28:41,376 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-10T20:28:41,376 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,34959,1731270520519-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:41,377 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,34959,1731270520519-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:41,377 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,34959,1731270520519-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:41,377 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-734d44df782b:34959, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:41,377 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:41,377 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:41,379 DEBUG [master/734d44df782b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T20:28:41,380 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.781sec 2024-11-10T20:28:41,381 INFO [master/734d44df782b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T20:28:41,381 INFO [master/734d44df782b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T20:28:41,381 INFO [master/734d44df782b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T20:28:41,381 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T20:28:41,381 INFO [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T20:28:41,381 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,34959,1731270520519-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:28:41,381 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,34959,1731270520519-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T20:28:41,384 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T20:28:41,384 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T20:28:41,384 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,34959,1731270520519-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:28:41,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac3c66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:28:41,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 734d44df782b,34959,-1 for getting cluster id 2024-11-10T20:28:41,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T20:28:41,484 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8928a194-422a-4e34-baef-d1b363868780' 2024-11-10T20:28:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T20:28:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8928a194-422a-4e34-baef-d1b363868780" 2024-11-10T20:28:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72a1ab84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:28:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [734d44df782b,34959,-1] 2024-11-10T20:28:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T20:28:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:28:41,486 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T20:28:41,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec76923, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:28:41,488 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:28:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,44249,1731270520567, seqNum=-1] 2024-11-10T20:28:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:28:41,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:28:41,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=734d44df782b,34959,1731270520519 2024-11-10T20:28:41,492 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:28:41,494 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T20:28:41,495 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T20:28:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 734d44df782b,34959,1731270520519 2024-11-10T20:28:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5652b532 2024-11-10T20:28:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T20:28:41,497 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T20:28:41,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T20:28:41,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T20:28:41,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:28:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:28:41,501 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T20:28:41,501 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:41,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-11-10T20:28:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:28:41,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T20:28:41,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741835_1011 (size=405) 2024-11-10T20:28:41,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741835_1011 (size=405) 2024-11-10T20:28:41,511 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e541f84b11318e691421fc29386eba7c, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2 2024-11-10T20:28:41,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741836_1012 (size=88) 2024-11-10T20:28:41,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741836_1012 (size=88) 2024-11-10T20:28:41,518 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:41,518 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing e541f84b11318e691421fc29386eba7c, disabling compactions & flushes 2024-11-10T20:28:41,518 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:41,518 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:41,518 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. after waiting 0 ms 2024-11-10T20:28:41,518 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:41,518 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:41,518 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for e541f84b11318e691421fc29386eba7c: Waiting for close lock at 1731270521518Disabling compacts and flushes for region at 1731270521518Disabling writes for close at 1731270521518Writing region close event to WAL at 1731270521518Closed at 1731270521518 2024-11-10T20:28:41,520 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T20:28:41,520 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731270521520"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731270521520"}]},"ts":"1731270521520"} 2024-11-10T20:28:41,523 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T20:28:41,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T20:28:41,524 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270521524"}]},"ts":"1731270521524"} 2024-11-10T20:28:41,526 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-11-10T20:28:41,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e541f84b11318e691421fc29386eba7c, ASSIGN}] 2024-11-10T20:28:41,528 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e541f84b11318e691421fc29386eba7c, ASSIGN 2024-11-10T20:28:41,529 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e541f84b11318e691421fc29386eba7c, ASSIGN; state=OFFLINE, location=734d44df782b,44249,1731270520567; forceNewPlan=false, retain=false 2024-11-10T20:28:41,680 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e541f84b11318e691421fc29386eba7c, regionState=OPENING, regionLocation=734d44df782b,44249,1731270520567 2024-11-10T20:28:41,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e541f84b11318e691421fc29386eba7c, ASSIGN because future has completed 2024-11-10T20:28:41,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e541f84b11318e691421fc29386eba7c, server=734d44df782b,44249,1731270520567}] 2024-11-10T20:28:41,839 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:41,839 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e541f84b11318e691421fc29386eba7c, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:28:41,839 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,839 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:28:41,839 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,840 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,841 INFO [StoreOpener-e541f84b11318e691421fc29386eba7c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,842 INFO [StoreOpener-e541f84b11318e691421fc29386eba7c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e541f84b11318e691421fc29386eba7c columnFamilyName info 2024-11-10T20:28:41,842 DEBUG [StoreOpener-e541f84b11318e691421fc29386eba7c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:28:41,842 INFO [StoreOpener-e541f84b11318e691421fc29386eba7c-1 {}] regionserver.HStore(327): Store=e541f84b11318e691421fc29386eba7c/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:28:41,843 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,843 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,843 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,844 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,844 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,845 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,847 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:28:41,847 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e541f84b11318e691421fc29386eba7c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=838852, jitterRate=0.06665553152561188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T20:28:41,848 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e541f84b11318e691421fc29386eba7c 2024-11-10T20:28:41,848 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e541f84b11318e691421fc29386eba7c: Running coprocessor pre-open hook at 1731270521840Writing region info on filesystem at 1731270521840Initializing all the Stores at 1731270521840Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270521840Cleaning up temporary data from old regions at 1731270521844 (+4 ms)Running coprocessor post-open hooks at 1731270521848 (+4 ms)Region opened successfully at 1731270521848 2024-11-10T20:28:41,849 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c., pid=6, masterSystemTime=1731270521835 2024-11-10T20:28:41,851 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:41,852 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:41,852 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e541f84b11318e691421fc29386eba7c, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,44249,1731270520567 2024-11-10T20:28:41,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e541f84b11318e691421fc29386eba7c, server=734d44df782b,44249,1731270520567 because future has completed 2024-11-10T20:28:41,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T20:28:41,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e541f84b11318e691421fc29386eba7c, server=734d44df782b,44249,1731270520567 in 174 msec 2024-11-10T20:28:41,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T20:28:41,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e541f84b11318e691421fc29386eba7c, ASSIGN in 332 msec 2024-11-10T20:28:41,862 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T20:28:41,862 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270521862"}]},"ts":"1731270521862"} 2024-11-10T20:28:41,864 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-11-10T20:28:41,865 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T20:28:41,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 367 msec 2024-11-10T20:28:42,197 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:42,197 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:42,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T20:28:42,804 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T20:28:42,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:28:42,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T20:28:42,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:28:42,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-11-10T20:28:43,198 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:43,198 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:44,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:44,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:45,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:45,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:46,200 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:46,200 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:46,838 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T20:28:46,839 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,839 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,839 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,840 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,840 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,840 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,856 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,856 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,856 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,857 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,857 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,857 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,860 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,861 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,861 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,863 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:28:46,870 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T20:28:46,870 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-11-10T20:28:47,200 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:47,200 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:48,201 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:48,201 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:49,202 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:49,202 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:50,202 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:50,202 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:51,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:51,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:28:51,582 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T20:28:51,582 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-11-10T20:28:51,585 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:28:51,585 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:51,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c., hostname=734d44df782b,44249,1731270520567, seqNum=2] 2024-11-10T20:28:51,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:28:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:28:51,601 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T20:28:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T20:28:51,602 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T20:28:51,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T20:28:51,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44249 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T20:28:51,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:51,764 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e541f84b11318e691421fc29386eba7c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T20:28:51,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/fe86145171914fd982e3f5092b1ccad8 is 1080, key is row0001/info:/1731270531589/Put/seqid=0 2024-11-10T20:28:51,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741837_1013 (size=6033) 2024-11-10T20:28:51,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741837_1013 (size=6033) 2024-11-10T20:28:51,786 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/fe86145171914fd982e3f5092b1ccad8 2024-11-10T20:28:51,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/fe86145171914fd982e3f5092b1ccad8 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/fe86145171914fd982e3f5092b1ccad8 2024-11-10T20:28:51,798 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/fe86145171914fd982e3f5092b1ccad8, entries=1, sequenceid=5, filesize=5.9 K 2024-11-10T20:28:51,799 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e541f84b11318e691421fc29386eba7c in 35ms, sequenceid=5, compaction requested=false 2024-11-10T20:28:51,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e541f84b11318e691421fc29386eba7c: 2024-11-10T20:28:51,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:28:51,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T20:28:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T20:28:51,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T20:28:51,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 201 msec 2024-11-10T20:28:51,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 211 msec 2024-11-10T20:28:52,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:52,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:53,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:53,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:54,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:54,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:55,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:55,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:56,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:56,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:57,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:57,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:58,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:58,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:59,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:28:59,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:00,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:00,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:01,209 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:01,209 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T20:29:01,662 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T20:29:01,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:29:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:29:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T20:29:01,667 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T20:29:01,668 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T20:29:01,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T20:29:01,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44249 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T20:29:01,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:01,822 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e541f84b11318e691421fc29386eba7c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T20:29:01,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/7965eeb2714c4a9fb162c1806fd9feb1 is 1080, key is row0002/info:/1731270541663/Put/seqid=0 2024-11-10T20:29:01,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741838_1014 (size=6033) 2024-11-10T20:29:01,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741838_1014 (size=6033) 2024-11-10T20:29:01,834 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/7965eeb2714c4a9fb162c1806fd9feb1 2024-11-10T20:29:01,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/7965eeb2714c4a9fb162c1806fd9feb1 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/7965eeb2714c4a9fb162c1806fd9feb1 2024-11-10T20:29:01,845 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/7965eeb2714c4a9fb162c1806fd9feb1, entries=1, sequenceid=9, filesize=5.9 K 2024-11-10T20:29:01,846 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e541f84b11318e691421fc29386eba7c in 24ms, sequenceid=9, compaction requested=false 2024-11-10T20:29:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e541f84b11318e691421fc29386eba7c: 2024-11-10T20:29:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T20:29:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T20:29:01,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T20:29:01,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 180 msec 2024-11-10T20:29:01,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 187 msec 2024-11-10T20:29:02,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:02,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:03,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:03,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:04,211 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:04,211 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:05,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:05,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:06,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:06,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:07,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:07,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:07,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 after 68047ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor192.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:29:07,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta after 68036ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor192.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T20:29:08,214 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:08,214 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:09,214 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:09,214 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:10,215 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:10,215 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:10,502 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T20:29:11,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:11,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T20:29:11,762 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T20:29:11,765 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C44249%2C1731270520567.1731270551765 2024-11-10T20:29:11,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:11,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:11,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:11,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:11,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:11,771 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270520945 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270551765 2024-11-10T20:29:11,772 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36873:36873),(127.0.0.1/127.0.0.1:39575:39575)] 2024-11-10T20:29:11,772 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270520945 is not closed yet, will try archiving it next time 2024-11-10T20:29:11,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:29:11,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741833_1009 (size=5546) 2024-11-10T20:29:11,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741833_1009 (size=5546) 2024-11-10T20:29:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:29:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T20:29:11,775 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T20:29:11,776 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T20:29:11,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T20:29:11,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44249 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T20:29:11,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:11,930 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e541f84b11318e691421fc29386eba7c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T20:29:11,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/2a6a4cb4f87e42afae4a7790f1084034 is 1080, key is row0003/info:/1731270551763/Put/seqid=0 2024-11-10T20:29:11,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741840_1016 (size=6033) 2024-11-10T20:29:11,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741840_1016 (size=6033) 2024-11-10T20:29:11,940 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/2a6a4cb4f87e42afae4a7790f1084034 2024-11-10T20:29:11,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/2a6a4cb4f87e42afae4a7790f1084034 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/2a6a4cb4f87e42afae4a7790f1084034 2024-11-10T20:29:11,951 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/2a6a4cb4f87e42afae4a7790f1084034, entries=1, sequenceid=13, filesize=5.9 K 2024-11-10T20:29:11,953 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e541f84b11318e691421fc29386eba7c in 22ms, sequenceid=13, compaction requested=true 2024-11-10T20:29:11,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e541f84b11318e691421fc29386eba7c: 2024-11-10T20:29:11,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:11,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T20:29:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T20:29:11,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T20:29:11,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 178 msec 2024-11-10T20:29:11,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 185 msec 2024-11-10T20:29:12,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:12,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:13,217 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:13,217 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:14,217 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:14,217 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:15,218 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:15,218 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:16,219 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:16,219 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:17,219 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:17,219 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:18,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:18,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:19,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:19,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:20,221 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:20,221 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:21,222 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:21,222 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:21,385 INFO [master/734d44df782b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T20:29:21,385 INFO [master/734d44df782b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T20:29:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T20:29:21,822 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T20:29:21,822 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:29:21,824 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:29:21,824 DEBUG [Time-limited test {}] regionserver.HStore(1541): e541f84b11318e691421fc29386eba7c/info is initiating minor compaction (all files) 2024-11-10T20:29:21,824 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T20:29:21,824 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:21,824 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e541f84b11318e691421fc29386eba7c/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:21,825 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/fe86145171914fd982e3f5092b1ccad8, hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/7965eeb2714c4a9fb162c1806fd9feb1, hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/2a6a4cb4f87e42afae4a7790f1084034] into tmpdir=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp, totalSize=17.7 K 2024-11-10T20:29:21,825 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fe86145171914fd982e3f5092b1ccad8, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731270531589 2024-11-10T20:29:21,825 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7965eeb2714c4a9fb162c1806fd9feb1, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731270541663 2024-11-10T20:29:21,826 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2a6a4cb4f87e42afae4a7790f1084034, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731270551763 2024-11-10T20:29:21,837 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e541f84b11318e691421fc29386eba7c#info#compaction#44 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:29:21,838 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/44c8073881614a808c262373ea04cb10 is 1080, key is row0001/info:/1731270531589/Put/seqid=0 2024-11-10T20:29:21,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741841_1017 (size=8296) 2024-11-10T20:29:21,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741841_1017 (size=8296) 2024-11-10T20:29:21,848 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/44c8073881614a808c262373ea04cb10 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/44c8073881614a808c262373ea04cb10 2024-11-10T20:29:21,854 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e541f84b11318e691421fc29386eba7c/info of e541f84b11318e691421fc29386eba7c into 44c8073881614a808c262373ea04cb10(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:29:21,854 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e541f84b11318e691421fc29386eba7c: 2024-11-10T20:29:21,856 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C44249%2C1731270520567.1731270561856 2024-11-10T20:29:21,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:21,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:21,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:21,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:21,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:21,862 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270551765 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270561856 2024-11-10T20:29:21,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741839_1015 (size=2520) 2024-11-10T20:29:21,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741839_1015 (size=2520) 2024-11-10T20:29:21,865 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270520945 to hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/oldWALs/734d44df782b%2C44249%2C1731270520567.1731270520945 2024-11-10T20:29:21,868 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36873:36873),(127.0.0.1/127.0.0.1:39575:39575)] 2024-11-10T20:29:21,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:29:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:29:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T20:29:21,871 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T20:29:21,872 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T20:29:21,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T20:29:22,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44249 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T20:29:22,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:22,026 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e541f84b11318e691421fc29386eba7c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T20:29:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/b208c41f2ef745f2aa2e987b530278ed is 1080, key is row0000/info:/1731270561855/Put/seqid=0 2024-11-10T20:29:22,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741843_1019 (size=6033) 2024-11-10T20:29:22,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741843_1019 (size=6033) 2024-11-10T20:29:22,036 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/b208c41f2ef745f2aa2e987b530278ed 2024-11-10T20:29:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/b208c41f2ef745f2aa2e987b530278ed as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/b208c41f2ef745f2aa2e987b530278ed 2024-11-10T20:29:22,046 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/b208c41f2ef745f2aa2e987b530278ed, entries=1, sequenceid=18, filesize=5.9 K 2024-11-10T20:29:22,047 INFO [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e541f84b11318e691421fc29386eba7c in 21ms, sequenceid=18, compaction requested=false 2024-11-10T20:29:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e541f84b11318e691421fc29386eba7c: 2024-11-10T20:29:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T20:29:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T20:29:22,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T20:29:22,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec 2024-11-10T20:29:22,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 183 msec 2024-11-10T20:29:22,222 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:22,222 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:23,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:23,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:24,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:24,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:25,224 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:25,224 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:26,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:26,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:26,839 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region e541f84b11318e691421fc29386eba7c, had cached 0 bytes from a total of 14329 2024-11-10T20:29:27,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:27,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:28,226 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:28,226 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:29,226 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:29,226 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:30,227 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:30,227 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:31,228 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:31,228 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T20:29:31,952 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T20:29:31,955 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C44249%2C1731270520567.1731270571955 2024-11-10T20:29:31,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:31,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:31,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:31,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:31,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:31,961 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270561856 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270571955 2024-11-10T20:29:31,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741842_1018 (size=2026) 2024-11-10T20:29:31,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741842_1018 (size=2026) 2024-11-10T20:29:31,964 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/WALs/734d44df782b,44249,1731270520567/734d44df782b%2C44249%2C1731270520567.1731270551765 to hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/oldWALs/734d44df782b%2C44249%2C1731270520567.1731270551765 2024-11-10T20:29:31,970 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36873:36873),(127.0.0.1/127.0.0.1:39575:39575)] 2024-11-10T20:29:31,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T20:29:31,970 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:29:31,970 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:29:31,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:29:31,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:29:31,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T20:29:31,970 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T20:29:31,970 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=90659830, stopped=false 2024-11-10T20:29:31,970 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=734d44df782b,34959,1731270520519 2024-11-10T20:29:31,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:29:31,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:29:31,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:31,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:31,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:29:31,972 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:29:31,973 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:29:31,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:29:31,973 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '734d44df782b,44249,1731270520567' ***** 2024-11-10T20:29:31,973 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T20:29:31,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:29:31,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:29:31,973 INFO [RS:0;734d44df782b:44249 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T20:29:31,973 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T20:29:31,973 INFO [RS:0;734d44df782b:44249 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(3091): Received CLOSE for e541f84b11318e691421fc29386eba7c 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(959): stopping server 734d44df782b,44249,1731270520567 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;734d44df782b:44249. 2024-11-10T20:29:31,974 DEBUG [RS:0;734d44df782b:44249 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:29:31,974 DEBUG [RS:0;734d44df782b:44249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:29:31,974 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e541f84b11318e691421fc29386eba7c, disabling compactions & flushes 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T20:29:31,974 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T20:29:31,974 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:31,974 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. after waiting 0 ms 2024-11-10T20:29:31,974 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:31,974 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T20:29:31,974 DEBUG [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e541f84b11318e691421fc29386eba7c=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.} 2024-11-10T20:29:31,974 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e541f84b11318e691421fc29386eba7c 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T20:29:31,974 DEBUG [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e541f84b11318e691421fc29386eba7c 2024-11-10T20:29:31,974 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:29:31,974 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:29:31,975 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:29:31,975 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:29:31,975 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:29:31,975 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-11-10T20:29:31,979 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/5e97cb4a779f4072bb7e45415fbbc9f2 is 1080, key is row0001/info:/1731270571953/Put/seqid=0 2024-11-10T20:29:31,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741845_1021 (size=6033) 2024-11-10T20:29:31,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741845_1021 (size=6033) 2024-11-10T20:29:31,984 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/5e97cb4a779f4072bb7e45415fbbc9f2 2024-11-10T20:29:31,990 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/.tmp/info/5e97cb4a779f4072bb7e45415fbbc9f2 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/5e97cb4a779f4072bb7e45415fbbc9f2 2024-11-10T20:29:31,991 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/.tmp/info/ce5a0f7ff8e84aadae3776b11e0a8293 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c./info:regioninfo/1731270521852/Put/seqid=0 2024-11-10T20:29:31,995 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/5e97cb4a779f4072bb7e45415fbbc9f2, entries=1, sequenceid=22, filesize=5.9 K 2024-11-10T20:29:31,996 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e541f84b11318e691421fc29386eba7c in 22ms, sequenceid=22, compaction requested=true 2024-11-10T20:29:31,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741846_1022 (size=7308) 2024-11-10T20:29:31,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741846_1022 (size=7308) 2024-11-10T20:29:31,997 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/fe86145171914fd982e3f5092b1ccad8, hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/7965eeb2714c4a9fb162c1806fd9feb1, hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/2a6a4cb4f87e42afae4a7790f1084034] to archive 2024-11-10T20:29:31,997 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/.tmp/info/ce5a0f7ff8e84aadae3776b11e0a8293 2024-11-10T20:29:31,998 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T20:29:31,999 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/fe86145171914fd982e3f5092b1ccad8 to hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/fe86145171914fd982e3f5092b1ccad8 2024-11-10T20:29:32,001 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/7965eeb2714c4a9fb162c1806fd9feb1 to hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/7965eeb2714c4a9fb162c1806fd9feb1 2024-11-10T20:29:32,002 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/2a6a4cb4f87e42afae4a7790f1084034 to hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/info/2a6a4cb4f87e42afae4a7790f1084034 2024-11-10T20:29:32,002 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=734d44df782b:34959 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-10T20:29:32,002 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [fe86145171914fd982e3f5092b1ccad8=6033, 7965eeb2714c4a9fb162c1806fd9feb1=6033, 2a6a4cb4f87e42afae4a7790f1084034=6033] 2024-11-10T20:29:32,006 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e541f84b11318e691421fc29386eba7c/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-11-10T20:29:32,006 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:32,006 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e541f84b11318e691421fc29386eba7c: Waiting for close lock at 1731270571974Running coprocessor pre-close hooks at 1731270571974Disabling compacts and flushes for region at 1731270571974Disabling writes for close at 1731270571974Obtaining lock to block concurrent updates at 1731270571974Preparing flush snapshotting stores in e541f84b11318e691421fc29386eba7c at 1731270571974Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731270571975 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. at 1731270571975Flushing e541f84b11318e691421fc29386eba7c/info: creating writer at 1731270571976 (+1 ms)Flushing e541f84b11318e691421fc29386eba7c/info: appending metadata at 1731270571978 (+2 ms)Flushing e541f84b11318e691421fc29386eba7c/info: closing flushed file at 1731270571978Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4872ea96: reopening flushed file at 1731270571989 (+11 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e541f84b11318e691421fc29386eba7c in 22ms, sequenceid=22, compaction requested=true at 1731270571996 (+7 ms)Writing region close event to WAL at 1731270572003 (+7 ms)Running coprocessor post-close hooks at 1731270572006 (+3 ms)Closed at 1731270572006 2024-11-10T20:29:32,006 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731270521497.e541f84b11318e691421fc29386eba7c. 2024-11-10T20:29:32,016 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/.tmp/ns/3353955ec1d9451eb0e09e994505ac60 is 43, key is default/ns:d/1731270521369/Put/seqid=0 2024-11-10T20:29:32,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741847_1023 (size=5153) 2024-11-10T20:29:32,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741847_1023 (size=5153) 2024-11-10T20:29:32,021 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/.tmp/ns/3353955ec1d9451eb0e09e994505ac60 2024-11-10T20:29:32,039 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/.tmp/table/94c974b0e09a421d81019a6ee01885df is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731270521862/Put/seqid=0 2024-11-10T20:29:32,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741848_1024 (size=5508) 2024-11-10T20:29:32,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741848_1024 (size=5508) 2024-11-10T20:29:32,044 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/.tmp/table/94c974b0e09a421d81019a6ee01885df 2024-11-10T20:29:32,049 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/.tmp/info/ce5a0f7ff8e84aadae3776b11e0a8293 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/info/ce5a0f7ff8e84aadae3776b11e0a8293 2024-11-10T20:29:32,053 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/info/ce5a0f7ff8e84aadae3776b11e0a8293, entries=10, sequenceid=11, filesize=7.1 K 2024-11-10T20:29:32,054 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/.tmp/ns/3353955ec1d9451eb0e09e994505ac60 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/ns/3353955ec1d9451eb0e09e994505ac60 2024-11-10T20:29:32,059 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/ns/3353955ec1d9451eb0e09e994505ac60, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T20:29:32,060 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/.tmp/table/94c974b0e09a421d81019a6ee01885df as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/table/94c974b0e09a421d81019a6ee01885df 2024-11-10T20:29:32,064 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/table/94c974b0e09a421d81019a6ee01885df, entries=2, sequenceid=11, filesize=5.4 K 2024-11-10T20:29:32,065 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=11, compaction requested=false 2024-11-10T20:29:32,069 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T20:29:32,070 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:29:32,070 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:29:32,070 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270571974Running coprocessor pre-close hooks at 1731270571974Disabling compacts and flushes for region at 1731270571974Disabling writes for close at 1731270571975 (+1 ms)Obtaining lock to block concurrent updates at 1731270571975Preparing flush snapshotting stores in 1588230740 at 1731270571975Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731270571975Flushing stores of hbase:meta,,1.1588230740 at 1731270571976 (+1 ms)Flushing 1588230740/info: creating writer at 1731270571976Flushing 1588230740/info: appending metadata at 1731270571991 (+15 ms)Flushing 1588230740/info: closing flushed file at 1731270571991Flushing 1588230740/ns: creating writer at 1731270572002 (+11 ms)Flushing 1588230740/ns: appending metadata at 1731270572015 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731270572015Flushing 1588230740/table: creating writer at 1731270572025 (+10 ms)Flushing 1588230740/table: appending metadata at 1731270572038 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731270572038Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36874289: reopening flushed file at 1731270572048 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e241758: reopening flushed file at 1731270572054 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bd861c0: reopening flushed file at 1731270572059 (+5 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=11, compaction requested=false at 1731270572065 (+6 ms)Writing region close event to WAL at 1731270572066 (+1 ms)Running coprocessor post-close hooks at 1731270572070 (+4 ms)Closed at 1731270572070 2024-11-10T20:29:32,070 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T20:29:32,175 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(976): stopping server 734d44df782b,44249,1731270520567; all regions closed. 2024-11-10T20:29:32,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741834_1010 (size=3306) 2024-11-10T20:29:32,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741834_1010 (size=3306) 2024-11-10T20:29:32,180 DEBUG [RS:0;734d44df782b:44249 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/oldWALs 2024-11-10T20:29:32,180 INFO [RS:0;734d44df782b:44249 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C44249%2C1731270520567.meta:.meta(num 1731270521329) 2024-11-10T20:29:32,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741844_1020 (size=1252) 2024-11-10T20:29:32,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741844_1020 (size=1252) 2024-11-10T20:29:32,185 DEBUG [RS:0;734d44df782b:44249 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/oldWALs 2024-11-10T20:29:32,185 INFO [RS:0;734d44df782b:44249 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C44249%2C1731270520567:(num 1731270571955) 2024-11-10T20:29:32,185 DEBUG [RS:0;734d44df782b:44249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:29:32,185 INFO [RS:0;734d44df782b:44249 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:29:32,185 INFO [RS:0;734d44df782b:44249 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:29:32,185 INFO [RS:0;734d44df782b:44249 {}] hbase.ChoreService(370): Chore service for: regionserver/734d44df782b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T20:29:32,186 INFO [RS:0;734d44df782b:44249 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:29:32,186 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:29:32,186 INFO [RS:0;734d44df782b:44249 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44249 2024-11-10T20:29:32,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:29:32,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/734d44df782b,44249,1731270520567 2024-11-10T20:29:32,188 INFO [RS:0;734d44df782b:44249 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:29:32,189 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [734d44df782b,44249,1731270520567] 2024-11-10T20:29:32,191 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/734d44df782b,44249,1731270520567 already deleted, retry=false 2024-11-10T20:29:32,191 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 734d44df782b,44249,1731270520567 expired; onlineServers=0 2024-11-10T20:29:32,191 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '734d44df782b,34959,1731270520519' ***** 2024-11-10T20:29:32,191 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T20:29:32,191 INFO [M:0;734d44df782b:34959 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:29:32,191 INFO [M:0;734d44df782b:34959 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:29:32,191 DEBUG [M:0;734d44df782b:34959 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T20:29:32,191 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T20:29:32,191 DEBUG [M:0;734d44df782b:34959 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T20:29:32,191 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270520729 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270520729,5,FailOnTimeoutGroup] 2024-11-10T20:29:32,191 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270520730 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270520730,5,FailOnTimeoutGroup] 2024-11-10T20:29:32,191 INFO [M:0;734d44df782b:34959 {}] hbase.ChoreService(370): Chore service for: master/734d44df782b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T20:29:32,192 INFO [M:0;734d44df782b:34959 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:29:32,192 DEBUG [M:0;734d44df782b:34959 {}] master.HMaster(1795): Stopping service threads 2024-11-10T20:29:32,192 INFO [M:0;734d44df782b:34959 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T20:29:32,192 INFO [M:0;734d44df782b:34959 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:29:32,192 INFO [M:0;734d44df782b:34959 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T20:29:32,192 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T20:29:32,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T20:29:32,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:32,193 DEBUG [M:0;734d44df782b:34959 {}] zookeeper.ZKUtil(347): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T20:29:32,193 WARN [M:0;734d44df782b:34959 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T20:29:32,194 INFO [M:0;734d44df782b:34959 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/.lastflushedseqids 2024-11-10T20:29:32,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741849_1025 (size=130) 2024-11-10T20:29:32,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741849_1025 (size=130) 2024-11-10T20:29:32,199 INFO [M:0;734d44df782b:34959 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T20:29:32,200 INFO [M:0;734d44df782b:34959 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T20:29:32,200 DEBUG [M:0;734d44df782b:34959 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:29:32,200 INFO [M:0;734d44df782b:34959 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:29:32,200 DEBUG [M:0;734d44df782b:34959 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:29:32,200 DEBUG [M:0;734d44df782b:34959 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:29:32,200 DEBUG [M:0;734d44df782b:34959 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:29:32,200 INFO [M:0;734d44df782b:34959 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.60 KB heapSize=55.01 KB 2024-11-10T20:29:32,216 DEBUG [M:0;734d44df782b:34959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5fa689a0d0ba4779b2b0277037e6cc3f is 82, key is hbase:meta,,1/info:regioninfo/1731270521353/Put/seqid=0 2024-11-10T20:29:32,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741850_1026 (size=5672) 2024-11-10T20:29:32,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741850_1026 (size=5672) 2024-11-10T20:29:32,221 INFO [M:0;734d44df782b:34959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5fa689a0d0ba4779b2b0277037e6cc3f 2024-11-10T20:29:32,228 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:32,228 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:32,239 DEBUG [M:0;734d44df782b:34959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6bfc5d2ddd9a42158abcbaa9722ed798 is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731270521866/Put/seqid=0 2024-11-10T20:29:32,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741851_1027 (size=7824) 2024-11-10T20:29:32,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741851_1027 (size=7824) 2024-11-10T20:29:32,244 INFO [M:0;734d44df782b:34959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.00 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6bfc5d2ddd9a42158abcbaa9722ed798 2024-11-10T20:29:32,249 INFO [M:0;734d44df782b:34959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6bfc5d2ddd9a42158abcbaa9722ed798 2024-11-10T20:29:32,263 DEBUG [M:0;734d44df782b:34959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d8b0d20d6966486a9fa3b408825e2b46 is 69, key is 734d44df782b,44249,1731270520567/rs:state/1731270520801/Put/seqid=0 2024-11-10T20:29:32,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741852_1028 (size=5156) 2024-11-10T20:29:32,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741852_1028 (size=5156) 2024-11-10T20:29:32,268 INFO [M:0;734d44df782b:34959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d8b0d20d6966486a9fa3b408825e2b46 2024-11-10T20:29:32,285 DEBUG [M:0;734d44df782b:34959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/06f3a424fce34d8ab4af7832af11fbf5 is 52, key is load_balancer_on/state:d/1731270521493/Put/seqid=0 2024-11-10T20:29:32,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741853_1029 (size=5056) 2024-11-10T20:29:32,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741853_1029 (size=5056) 2024-11-10T20:29:32,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:29:32,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x10193df24200001, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:29:32,290 INFO [RS:0;734d44df782b:44249 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:29:32,290 INFO [RS:0;734d44df782b:44249 {}] regionserver.HRegionServer(1031): Exiting; stopping=734d44df782b,44249,1731270520567; zookeeper connection closed. 2024-11-10T20:29:32,290 INFO [M:0;734d44df782b:34959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/06f3a424fce34d8ab4af7832af11fbf5 2024-11-10T20:29:32,290 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3596178 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3596178 2024-11-10T20:29:32,290 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T20:29:32,295 DEBUG [M:0;734d44df782b:34959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5fa689a0d0ba4779b2b0277037e6cc3f as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5fa689a0d0ba4779b2b0277037e6cc3f 2024-11-10T20:29:32,299 INFO [M:0;734d44df782b:34959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5fa689a0d0ba4779b2b0277037e6cc3f, entries=8, sequenceid=121, filesize=5.5 K 2024-11-10T20:29:32,300 DEBUG [M:0;734d44df782b:34959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6bfc5d2ddd9a42158abcbaa9722ed798 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6bfc5d2ddd9a42158abcbaa9722ed798 2024-11-10T20:29:32,305 INFO [M:0;734d44df782b:34959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6bfc5d2ddd9a42158abcbaa9722ed798 2024-11-10T20:29:32,305 INFO [M:0;734d44df782b:34959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6bfc5d2ddd9a42158abcbaa9722ed798, entries=14, sequenceid=121, filesize=7.6 K 2024-11-10T20:29:32,306 DEBUG [M:0;734d44df782b:34959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d8b0d20d6966486a9fa3b408825e2b46 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d8b0d20d6966486a9fa3b408825e2b46 2024-11-10T20:29:32,310 INFO [M:0;734d44df782b:34959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d8b0d20d6966486a9fa3b408825e2b46, entries=1, sequenceid=121, filesize=5.0 K 2024-11-10T20:29:32,311 DEBUG [M:0;734d44df782b:34959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/06f3a424fce34d8ab4af7832af11fbf5 as hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/06f3a424fce34d8ab4af7832af11fbf5 2024-11-10T20:29:32,316 INFO [M:0;734d44df782b:34959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35011/user/jenkins/test-data/b370be6c-8b30-9183-ac35-07e3121820a2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/06f3a424fce34d8ab4af7832af11fbf5, entries=1, sequenceid=121, filesize=4.9 K 2024-11-10T20:29:32,317 INFO [M:0;734d44df782b:34959 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.60 KB/44650, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false 2024-11-10T20:29:32,318 INFO [M:0;734d44df782b:34959 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:29:32,318 DEBUG [M:0;734d44df782b:34959 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270572200Disabling compacts and flushes for region at 1731270572200Disabling writes for close at 1731270572200Obtaining lock to block concurrent updates at 1731270572200Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731270572200Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44650, getHeapSize=56264, getOffHeapSize=0, getCellsCount=140 at 1731270572200Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731270572201 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731270572201Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731270572215 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731270572215Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731270572225 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731270572239 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731270572239Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731270572249 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731270572263 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731270572263Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731270572272 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731270572285 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731270572285Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76c350f3: reopening flushed file at 1731270572294 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58b3494e: reopening flushed file at 1731270572300 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21048a2b: reopening flushed file at 1731270572305 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b501540: reopening flushed file at 1731270572311 (+6 ms)Finished flush of dataSize ~43.60 KB/44650, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false at 1731270572317 (+6 ms)Writing region close event to WAL at 1731270572318 (+1 ms)Closed at 1731270572318 2024-11-10T20:29:32,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:29:32,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42043 is added to blk_1073741830_1006 (size=53047) 2024-11-10T20:29:32,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741830_1006 (size=53047) 2024-11-10T20:29:32,321 INFO [M:0;734d44df782b:34959 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T20:29:32,321 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:29:32,321 INFO [M:0;734d44df782b:34959 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34959 2024-11-10T20:29:32,322 INFO [M:0;734d44df782b:34959 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:29:32,423 INFO [M:0;734d44df782b:34959 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:29:32,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:29:32,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34959-0x10193df24200000, quorum=127.0.0.1:53833, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:29:32,426 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@374dfdaf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:29:32,426 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@46a495b0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:29:32,426 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:29:32,426 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47d3f616{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:29:32,426 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c020752{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.log.dir/,STOPPED} 2024-11-10T20:29:32,428 WARN [BP-484077747-172.17.0.2-1731270519856 heartbeating to localhost/127.0.0.1:35011 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:29:32,428 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:29:32,428 WARN [BP-484077747-172.17.0.2-1731270519856 heartbeating to localhost/127.0.0.1:35011 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-484077747-172.17.0.2-1731270519856 (Datanode Uuid ba12c7d2-f7f6-42c9-9e36-65b58ff0ff90) service to localhost/127.0.0.1:35011 2024-11-10T20:29:32,428 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:29:32,428 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/data/data3/current/BP-484077747-172.17.0.2-1731270519856 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:29:32,429 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/data/data4/current/BP-484077747-172.17.0.2-1731270519856 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:29:32,429 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:29:32,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@68744dfe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:29:32,431 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6eebb3dc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:29:32,431 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:29:32,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@461cdaba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:29:32,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4eb40860{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.log.dir/,STOPPED} 2024-11-10T20:29:32,433 WARN [BP-484077747-172.17.0.2-1731270519856 heartbeating to localhost/127.0.0.1:35011 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:29:32,433 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:29:32,433 WARN [BP-484077747-172.17.0.2-1731270519856 heartbeating to localhost/127.0.0.1:35011 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-484077747-172.17.0.2-1731270519856 (Datanode Uuid 95c6cd7c-cd6c-4b11-adec-eadbb917afa4) service to localhost/127.0.0.1:35011 2024-11-10T20:29:32,433 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:29:32,433 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/data/data1/current/BP-484077747-172.17.0.2-1731270519856 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:29:32,433 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/cluster_05db0b53-d316-0dca-f1f9-dc5a30b45e00/data/data2/current/BP-484077747-172.17.0.2-1731270519856 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:29:32,434 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:29:32,440 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@521d1c5a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:29:32,440 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@62e6de45{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:29:32,440 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:29:32,440 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13b3ff43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:29:32,440 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62ee2d8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.log.dir/,STOPPED} 2024-11-10T20:29:32,446 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T20:29:32,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T20:29:32,471 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 179) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35011 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-36-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35011 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-35-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35011 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35011 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:35011 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35011 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:35011 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35011 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35011 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=28 (was 18) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7265 (was 7331) 2024-11-10T20:29:32,479 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=206, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=28, ProcessCount=11, AvailableMemoryMB=7265 2024-11-10T20:29:32,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T20:29:32,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.log.dir so I do NOT create it in target/test-data/86721941-e86f-274c-b4d3-27be669a4775 2024-11-10T20:29:32,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4a9afa9b-5291-724c-0157-d2a552de0c98/hadoop.tmp.dir so I do NOT create it in target/test-data/86721941-e86f-274c-b4d3-27be669a4775 2024-11-10T20:29:32,479 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984, deleteOnExit=true 2024-11-10T20:29:32,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T20:29:32,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/test.cache.data in system properties and HBase conf 2024-11-10T20:29:32,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.log.dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T20:29:32,480 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:29:32,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:29:32,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T20:29:32,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/nfs.dump.dir in system properties and HBase conf 2024-11-10T20:29:32,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/java.io.tmpdir in system properties and HBase conf 2024-11-10T20:29:32,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:29:32,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T20:29:32,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T20:29:32,493 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:29:32,563 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:29:32,568 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:29:32,569 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:29:32,569 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:29:32,569 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:29:32,569 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:29:32,570 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70e7b313{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:29:32,570 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7113b5b4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:29:32,685 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@11e6950{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/java.io.tmpdir/jetty-localhost-46175-hadoop-hdfs-3_4_1-tests_jar-_-any-11896664061950395482/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:29:32,685 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53fc2fb7{HTTP/1.1, (http/1.1)}{localhost:46175} 2024-11-10T20:29:32,686 INFO [Time-limited test {}] server.Server(415): Started @235160ms 2024-11-10T20:29:32,698 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:29:32,760 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:29:32,763 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:29:32,763 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:29:32,763 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:29:32,764 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:29:32,764 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@319ddf98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:29:32,764 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@362b84a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:29:32,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:29:32,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T20:29:32,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T20:29:32,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T20:29:32,814 INFO [regionserver/734d44df782b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:29:32,879 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10e59f6e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/java.io.tmpdir/jetty-localhost-38117-hadoop-hdfs-3_4_1-tests_jar-_-any-1179200814954586613/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:29:32,879 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@256ffd79{HTTP/1.1, (http/1.1)}{localhost:38117} 2024-11-10T20:29:32,879 INFO [Time-limited test {}] server.Server(415): Started @235353ms 2024-11-10T20:29:32,881 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:29:32,908 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:29:32,911 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:29:32,912 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:29:32,912 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:29:32,912 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:29:32,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45bda0cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:29:32,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4881a2ed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:29:32,970 WARN [Thread-1949 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/data/data1/current/BP-1481773930-172.17.0.2-1731270572511/current, will proceed with Du for space computation calculation, 2024-11-10T20:29:32,970 WARN [Thread-1950 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/data/data2/current/BP-1481773930-172.17.0.2-1731270572511/current, will proceed with Du for space computation calculation, 2024-11-10T20:29:32,986 WARN [Thread-1928 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:29:32,988 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe732333f9dfe98f0 with lease ID 0xeaede5a2a21dcf67: Processing first storage report for DS-8e119827-2b1a-4748-b02d-8881b8920fa4 from datanode DatanodeRegistration(127.0.0.1:34313, datanodeUuid=6f6016a4-7b89-4587-80db-4865e4c81ea7, infoPort=42933, infoSecurePort=0, ipcPort=38793, storageInfo=lv=-57;cid=testClusterID;nsid=1557249561;c=1731270572511) 2024-11-10T20:29:32,988 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe732333f9dfe98f0 with lease ID 0xeaede5a2a21dcf67: from storage DS-8e119827-2b1a-4748-b02d-8881b8920fa4 node DatanodeRegistration(127.0.0.1:34313, datanodeUuid=6f6016a4-7b89-4587-80db-4865e4c81ea7, infoPort=42933, infoSecurePort=0, ipcPort=38793, storageInfo=lv=-57;cid=testClusterID;nsid=1557249561;c=1731270572511), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:29:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe732333f9dfe98f0 with lease ID 0xeaede5a2a21dcf67: Processing first storage report for DS-c729cbe7-606d-4c21-8eef-67381164ab36 from datanode DatanodeRegistration(127.0.0.1:34313, datanodeUuid=6f6016a4-7b89-4587-80db-4865e4c81ea7, infoPort=42933, infoSecurePort=0, ipcPort=38793, storageInfo=lv=-57;cid=testClusterID;nsid=1557249561;c=1731270572511) 2024-11-10T20:29:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe732333f9dfe98f0 with lease ID 0xeaede5a2a21dcf67: from storage DS-c729cbe7-606d-4c21-8eef-67381164ab36 node DatanodeRegistration(127.0.0.1:34313, datanodeUuid=6f6016a4-7b89-4587-80db-4865e4c81ea7, infoPort=42933, infoSecurePort=0, ipcPort=38793, storageInfo=lv=-57;cid=testClusterID;nsid=1557249561;c=1731270572511), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:29:33,027 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@20d96a0d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/java.io.tmpdir/jetty-localhost-33563-hadoop-hdfs-3_4_1-tests_jar-_-any-6680998812425648585/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:29:33,027 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4ac7d52f{HTTP/1.1, (http/1.1)}{localhost:33563} 2024-11-10T20:29:33,027 INFO [Time-limited test {}] server.Server(415): Started @235501ms 2024-11-10T20:29:33,028 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:29:33,117 WARN [Thread-1975 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/data/data3/current/BP-1481773930-172.17.0.2-1731270572511/current, will proceed with Du for space computation calculation, 2024-11-10T20:29:33,117 WARN [Thread-1976 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/data/data4/current/BP-1481773930-172.17.0.2-1731270572511/current, will proceed with Du for space computation calculation, 2024-11-10T20:29:33,133 WARN [Thread-1964 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:29:33,135 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x94d101445871e895 with lease ID 0xeaede5a2a21dcf68: Processing first storage report for DS-7ca0c250-1dfb-4209-a0ce-0742ee57bd91 from datanode DatanodeRegistration(127.0.0.1:45981, datanodeUuid=ba5f09c2-703e-45d2-9d0e-db807413166b, infoPort=35135, infoSecurePort=0, ipcPort=46007, storageInfo=lv=-57;cid=testClusterID;nsid=1557249561;c=1731270572511) 2024-11-10T20:29:33,135 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x94d101445871e895 with lease ID 0xeaede5a2a21dcf68: from storage DS-7ca0c250-1dfb-4209-a0ce-0742ee57bd91 node DatanodeRegistration(127.0.0.1:45981, datanodeUuid=ba5f09c2-703e-45d2-9d0e-db807413166b, infoPort=35135, infoSecurePort=0, ipcPort=46007, storageInfo=lv=-57;cid=testClusterID;nsid=1557249561;c=1731270572511), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:29:33,135 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x94d101445871e895 with lease ID 0xeaede5a2a21dcf68: Processing first storage report for DS-6ae33392-d48d-4f36-85e3-ed3fde820322 from datanode DatanodeRegistration(127.0.0.1:45981, datanodeUuid=ba5f09c2-703e-45d2-9d0e-db807413166b, infoPort=35135, infoSecurePort=0, ipcPort=46007, storageInfo=lv=-57;cid=testClusterID;nsid=1557249561;c=1731270572511) 2024-11-10T20:29:33,135 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x94d101445871e895 with lease ID 0xeaede5a2a21dcf68: from storage DS-6ae33392-d48d-4f36-85e3-ed3fde820322 node DatanodeRegistration(127.0.0.1:45981, datanodeUuid=ba5f09c2-703e-45d2-9d0e-db807413166b, infoPort=35135, infoSecurePort=0, ipcPort=46007, storageInfo=lv=-57;cid=testClusterID;nsid=1557249561;c=1731270572511), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:29:33,149 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775 2024-11-10T20:29:33,151 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/zookeeper_0, clientPort=58834, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T20:29:33,152 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58834 2024-11-10T20:29:33,152 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:29:33,154 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:29:33,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:29:33,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:29:33,163 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868 with version=8 2024-11-10T20:29:33,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/hbase-staging 2024-11-10T20:29:33,165 INFO [Time-limited test {}] client.ConnectionUtils(128): master/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:29:33,165 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:29:33,165 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:29:33,165 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:29:33,165 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:29:33,165 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:29:33,165 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T20:29:33,165 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:29:33,165 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35643 2024-11-10T20:29:33,166 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35643 connecting to ZooKeeper ensemble=127.0.0.1:58834 2024-11-10T20:29:33,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:356430x0, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:29:33,173 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35643-0x10193dff1c70000 connected 2024-11-10T20:29:33,190 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:29:33,192 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:29:33,193 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:29:33,194 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868, hbase.cluster.distributed=false 2024-11-10T20:29:33,195 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:29:33,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35643 2024-11-10T20:29:33,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35643 2024-11-10T20:29:33,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35643 2024-11-10T20:29:33,197 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35643 2024-11-10T20:29:33,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35643 2024-11-10T20:29:33,213 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:29:33,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:29:33,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:29:33,213 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:29:33,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:29:33,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:29:33,213 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T20:29:33,214 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:29:33,214 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44413 2024-11-10T20:29:33,215 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44413 connecting to ZooKeeper ensemble=127.0.0.1:58834 2024-11-10T20:29:33,216 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:29:33,217 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:29:33,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444130x0, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:29:33,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:444130x0, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:29:33,221 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44413-0x10193dff1c70001 connected 2024-11-10T20:29:33,221 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T20:29:33,222 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T20:29:33,222 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T20:29:33,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:29:33,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44413 2024-11-10T20:29:33,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44413 2024-11-10T20:29:33,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44413 2024-11-10T20:29:33,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44413 2024-11-10T20:29:33,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44413 2024-11-10T20:29:33,229 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:33,229 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:33,236 DEBUG [M:0;734d44df782b:35643 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;734d44df782b:35643 2024-11-10T20:29:33,236 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/734d44df782b,35643,1731270573164 2024-11-10T20:29:33,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:29:33,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:29:33,238 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/734d44df782b,35643,1731270573164 2024-11-10T20:29:33,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T20:29:33,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,240 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T20:29:33,240 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/734d44df782b,35643,1731270573164 from backup master directory 2024-11-10T20:29:33,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/734d44df782b,35643,1731270573164 2024-11-10T20:29:33,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:29:33,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:29:33,242 WARN [master/734d44df782b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:29:33,242 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=734d44df782b,35643,1731270573164 2024-11-10T20:29:33,246 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/hbase.id] with ID: ac0879a3-bb74-443d-9a2d-266e4c02a5ee 2024-11-10T20:29:33,246 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/.tmp/hbase.id 2024-11-10T20:29:33,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:29:33,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:29:33,252 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/.tmp/hbase.id]:[hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/hbase.id] 2024-11-10T20:29:33,263 INFO [master/734d44df782b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:29:33,263 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T20:29:33,264 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T20:29:33,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:29:33,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:29:33,273 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:29:33,274 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T20:29:33,274 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:29:33,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:29:33,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:29:33,280 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store 2024-11-10T20:29:33,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:29:33,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:29:33,286 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:29:33,286 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:29:33,286 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:29:33,286 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:29:33,287 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-10T20:29:33,287 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:29:33,287 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:29:33,287 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270573286Disabling compacts and flushes for region at 1731270573286Disabling writes for close at 1731270573287 (+1 ms)Writing region close event to WAL at 1731270573287Closed at 1731270573287 2024-11-10T20:29:33,287 WARN [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/.initializing 2024-11-10T20:29:33,287 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/WALs/734d44df782b,35643,1731270573164 2024-11-10T20:29:33,290 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C35643%2C1731270573164, suffix=, logDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/WALs/734d44df782b,35643,1731270573164, archiveDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/oldWALs, maxLogs=10 2024-11-10T20:29:33,290 INFO [master/734d44df782b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C35643%2C1731270573164.1731270573290 2024-11-10T20:29:33,296 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/WALs/734d44df782b,35643,1731270573164/734d44df782b%2C35643%2C1731270573164.1731270573290 2024-11-10T20:29:33,297 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42933:42933),(127.0.0.1/127.0.0.1:35135:35135)] 2024-11-10T20:29:33,297 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:29:33,297 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:29:33,297 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,297 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,298 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,299 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T20:29:33,300 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,300 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:29:33,300 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T20:29:33,301 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:29:33,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,302 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T20:29:33,302 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,303 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:29:33,303 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,304 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T20:29:33,304 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,304 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:29:33,304 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,305 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,305 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,306 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,306 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,306 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T20:29:33,307 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:29:33,309 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:29:33,309 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=690023, jitterRate=-0.12259092926979065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T20:29:33,310 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731270573297Initializing all the Stores at 1731270573298 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270573298Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270573298Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270573298Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270573298Cleaning up temporary data from old regions at 1731270573306 (+8 ms)Region opened successfully at 1731270573310 (+4 ms) 2024-11-10T20:29:33,310 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T20:29:33,313 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea05d88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:29:33,313 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T20:29:33,314 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T20:29:33,314 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T20:29:33,314 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T20:29:33,314 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T20:29:33,314 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T20:29:33,314 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T20:29:33,316 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T20:29:33,317 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T20:29:33,318 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T20:29:33,319 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T20:29:33,319 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T20:29:33,321 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T20:29:33,321 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T20:29:33,322 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T20:29:33,323 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T20:29:33,324 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T20:29:33,325 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T20:29:33,326 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T20:29:33,328 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T20:29:33,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:29:33,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:29:33,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,332 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=734d44df782b,35643,1731270573164, sessionid=0x10193dff1c70000, setting cluster-up flag (Was=false) 2024-11-10T20:29:33,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,339 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T20:29:33,340 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,35643,1731270573164 2024-11-10T20:29:33,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,349 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T20:29:33,350 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,35643,1731270573164 2024-11-10T20:29:33,351 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T20:29:33,352 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T20:29:33,353 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T20:29:33,353 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T20:29:33,353 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 734d44df782b,35643,1731270573164 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T20:29:33,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:29:33,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:29:33,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:29:33,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:29:33,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/734d44df782b:0, corePoolSize=10, maxPoolSize=10 2024-11-10T20:29:33,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:29:33,354 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,355 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731270603355 2024-11-10T20:29:33,355 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T20:29:33,355 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T20:29:33,355 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T20:29:33,355 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T20:29:33,355 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T20:29:33,356 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T20:29:33,356 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,356 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:29:33,356 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T20:29:33,356 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T20:29:33,356 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T20:29:33,356 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T20:29:33,356 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T20:29:33,356 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T20:29:33,357 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,357 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T20:29:33,360 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270573356,5,FailOnTimeoutGroup] 2024-11-10T20:29:33,360 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270573360,5,FailOnTimeoutGroup] 2024-11-10T20:29:33,360 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,360 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T20:29:33,360 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,360 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:29:33,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:29:33,369 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T20:29:33,370 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868 2024-11-10T20:29:33,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:29:33,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:29:33,376 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:29:33,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:29:33,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:29:33,379 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:29:33,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:29:33,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:29:33,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:29:33,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:29:33,382 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:29:33,382 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:29:33,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:29:33,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:29:33,383 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:29:33,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:29:33,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740 2024-11-10T20:29:33,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740 2024-11-10T20:29:33,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:29:33,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:29:33,385 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:29:33,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:29:33,388 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:29:33,388 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=790768, jitterRate=0.005513787269592285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:29:33,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731270573376Initializing all the Stores at 1731270573377 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270573377Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270573377Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270573377Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270573377Cleaning up temporary data from old regions at 1731270573385 (+8 ms)Region opened successfully at 1731270573389 (+4 ms) 2024-11-10T20:29:33,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:29:33,389 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:29:33,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:29:33,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:29:33,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:29:33,389 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:29:33,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270573389Disabling compacts and flushes for region at 1731270573389Disabling writes for close at 1731270573389Writing region close event to WAL at 1731270573389Closed at 1731270573389 2024-11-10T20:29:33,390 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:29:33,390 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T20:29:33,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T20:29:33,392 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:29:33,393 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T20:29:33,426 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(746): ClusterId : ac0879a3-bb74-443d-9a2d-266e4c02a5ee 2024-11-10T20:29:33,427 DEBUG [RS:0;734d44df782b:44413 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T20:29:33,429 DEBUG [RS:0;734d44df782b:44413 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T20:29:33,429 DEBUG [RS:0;734d44df782b:44413 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T20:29:33,431 DEBUG [RS:0;734d44df782b:44413 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T20:29:33,431 DEBUG [RS:0;734d44df782b:44413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685abc8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:29:33,443 DEBUG [RS:0;734d44df782b:44413 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;734d44df782b:44413 2024-11-10T20:29:33,443 INFO [RS:0;734d44df782b:44413 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T20:29:33,443 INFO [RS:0;734d44df782b:44413 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T20:29:33,443 DEBUG [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T20:29:33,444 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(2659): reportForDuty to master=734d44df782b,35643,1731270573164 with port=44413, startcode=1731270573213 2024-11-10T20:29:33,444 DEBUG [RS:0;734d44df782b:44413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T20:29:33,446 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58639, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T20:29:33,446 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35643 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 734d44df782b,44413,1731270573213 2024-11-10T20:29:33,446 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35643 {}] master.ServerManager(517): Registering regionserver=734d44df782b,44413,1731270573213 2024-11-10T20:29:33,448 DEBUG [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868 2024-11-10T20:29:33,448 DEBUG [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36749 2024-11-10T20:29:33,448 DEBUG [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T20:29:33,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:29:33,450 DEBUG [RS:0;734d44df782b:44413 {}] zookeeper.ZKUtil(111): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/734d44df782b,44413,1731270573213 2024-11-10T20:29:33,450 WARN [RS:0;734d44df782b:44413 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:29:33,450 INFO [RS:0;734d44df782b:44413 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:29:33,450 DEBUG [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213 2024-11-10T20:29:33,451 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [734d44df782b,44413,1731270573213] 2024-11-10T20:29:33,454 INFO [RS:0;734d44df782b:44413 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T20:29:33,475 INFO [RS:0;734d44df782b:44413 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T20:29:33,475 INFO [RS:0;734d44df782b:44413 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T20:29:33,476 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,476 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T20:29:33,476 INFO [RS:0;734d44df782b:44413 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T20:29:33,477 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:29:33,477 DEBUG [RS:0;734d44df782b:44413 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:29:33,477 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,477 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,477 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,477 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,477 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,478 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,44413,1731270573213-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:29:33,492 INFO [RS:0;734d44df782b:44413 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T20:29:33,492 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,44413,1731270573213-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,492 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,492 INFO [RS:0;734d44df782b:44413 {}] regionserver.Replication(171): 734d44df782b,44413,1731270573213 started 2024-11-10T20:29:33,506 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:33,506 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(1482): Serving as 734d44df782b,44413,1731270573213, RpcServer on 734d44df782b/172.17.0.2:44413, sessionid=0x10193dff1c70001 2024-11-10T20:29:33,506 DEBUG [RS:0;734d44df782b:44413 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T20:29:33,506 DEBUG [RS:0;734d44df782b:44413 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 734d44df782b,44413,1731270573213 2024-11-10T20:29:33,506 DEBUG [RS:0;734d44df782b:44413 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,44413,1731270573213' 2024-11-10T20:29:33,506 DEBUG [RS:0;734d44df782b:44413 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T20:29:33,506 DEBUG [RS:0;734d44df782b:44413 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T20:29:33,507 DEBUG [RS:0;734d44df782b:44413 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T20:29:33,507 DEBUG [RS:0;734d44df782b:44413 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T20:29:33,507 DEBUG [RS:0;734d44df782b:44413 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 734d44df782b,44413,1731270573213 2024-11-10T20:29:33,507 DEBUG [RS:0;734d44df782b:44413 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,44413,1731270573213' 2024-11-10T20:29:33,507 DEBUG [RS:0;734d44df782b:44413 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T20:29:33,507 DEBUG [RS:0;734d44df782b:44413 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T20:29:33,508 DEBUG [RS:0;734d44df782b:44413 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T20:29:33,508 INFO [RS:0;734d44df782b:44413 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T20:29:33,508 INFO [RS:0;734d44df782b:44413 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T20:29:33,543 WARN [734d44df782b:35643 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T20:29:33,609 INFO [RS:0;734d44df782b:44413 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C44413%2C1731270573213, suffix=, logDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213, archiveDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/oldWALs, maxLogs=32 2024-11-10T20:29:33,610 INFO [RS:0;734d44df782b:44413 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C44413%2C1731270573213.1731270573610 2024-11-10T20:29:33,615 INFO [RS:0;734d44df782b:44413 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.1731270573610 2024-11-10T20:29:33,616 DEBUG [RS:0;734d44df782b:44413 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42933:42933),(127.0.0.1/127.0.0.1:35135:35135)] 2024-11-10T20:29:33,793 DEBUG [734d44df782b:35643 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T20:29:33,794 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=734d44df782b,44413,1731270573213 2024-11-10T20:29:33,795 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,44413,1731270573213, state=OPENING 2024-11-10T20:29:33,797 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T20:29:33,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:29:33,799 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:29:33,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,44413,1731270573213}] 2024-11-10T20:29:33,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:29:33,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:29:33,952 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T20:29:33,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59525, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T20:29:33,958 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T20:29:33,958 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:29:33,959 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C44413%2C1731270573213.meta, suffix=.meta, logDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213, archiveDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/oldWALs, maxLogs=32 2024-11-10T20:29:33,960 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C44413%2C1731270573213.meta.1731270573959.meta 2024-11-10T20:29:33,964 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.meta.1731270573959.meta 2024-11-10T20:29:33,965 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35135:35135),(127.0.0.1/127.0.0.1:42933:42933)] 2024-11-10T20:29:33,966 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:29:33,966 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T20:29:33,966 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T20:29:33,966 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T20:29:33,966 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T20:29:33,966 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:29:33,966 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T20:29:33,967 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T20:29:33,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:29:33,968 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:29:33,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:29:33,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:29:33,969 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:29:33,969 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:29:33,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:29:33,970 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:29:33,970 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:29:33,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:29:33,971 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:29:33,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:33,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:29:33,972 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:29:33,973 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740 2024-11-10T20:29:33,974 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740 2024-11-10T20:29:33,975 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:29:33,975 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:29:33,975 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:29:33,977 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:29:33,978 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=741769, jitterRate=-0.05679278075695038}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:29:33,978 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T20:29:33,979 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731270573967Writing region info on filesystem at 1731270573967Initializing all the Stores at 1731270573967Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270573967Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270573967Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270573967Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270573967Cleaning up temporary data from old regions at 1731270573975 (+8 ms)Running coprocessor post-open hooks at 1731270573978 (+3 ms)Region opened successfully at 1731270573978 2024-11-10T20:29:33,980 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731270573952 2024-11-10T20:29:33,982 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T20:29:33,982 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T20:29:33,983 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,44413,1731270573213 2024-11-10T20:29:33,984 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,44413,1731270573213, state=OPEN 2024-11-10T20:29:33,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:29:33,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:29:33,988 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=734d44df782b,44413,1731270573213 2024-11-10T20:29:33,988 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:29:33,988 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:29:33,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T20:29:33,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,44413,1731270573213 in 189 msec 2024-11-10T20:29:33,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T20:29:33,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 601 msec 2024-11-10T20:29:33,994 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:29:33,994 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T20:29:33,995 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:29:33,996 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,44413,1731270573213, seqNum=-1] 2024-11-10T20:29:33,996 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:29:33,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48861, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:29:34,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 649 msec 2024-11-10T20:29:34,002 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731270574002, completionTime=-1 2024-11-10T20:29:34,002 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T20:29:34,002 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731270634004 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731270694004 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,35643,1731270573164-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,35643,1731270573164-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,35643,1731270573164-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-734d44df782b:35643, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:34,004 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:34,006 DEBUG [master/734d44df782b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T20:29:34,007 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.765sec 2024-11-10T20:29:34,007 INFO [master/734d44df782b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T20:29:34,008 INFO [master/734d44df782b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T20:29:34,008 INFO [master/734d44df782b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T20:29:34,008 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T20:29:34,008 INFO [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T20:29:34,008 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,35643,1731270573164-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:29:34,008 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,35643,1731270573164-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T20:29:34,010 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T20:29:34,010 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T20:29:34,010 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,35643,1731270573164-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:29:34,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21284ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:29:34,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 734d44df782b,35643,-1 for getting cluster id 2024-11-10T20:29:34,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T20:29:34,028 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac0879a3-bb74-443d-9a2d-266e4c02a5ee' 2024-11-10T20:29:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T20:29:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac0879a3-bb74-443d-9a2d-266e4c02a5ee" 2024-11-10T20:29:34,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6eae0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:29:34,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [734d44df782b,35643,-1] 2024-11-10T20:29:34,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T20:29:34,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:29:34,030 INFO [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T20:29:34,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4445ac53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:29:34,031 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:29:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,44413,1731270573213, seqNum=-1] 2024-11-10T20:29:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:29:34,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:29:34,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=734d44df782b,35643,1731270573164 2024-11-10T20:29:34,034 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:29:34,037 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T20:29:34,037 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T20:29:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 734d44df782b,35643,1731270573164 2024-11-10T20:29:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42021c82 2024-11-10T20:29:34,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T20:29:34,039 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T20:29:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35643 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T20:29:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T20:29:34,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:29:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35643 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-11-10T20:29:34,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T20:29:34,042 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:34,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-11-10T20:29:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:29:34,043 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T20:29:34,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741835_1011 (size=381) 2024-11-10T20:29:34,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741835_1011 (size=381) 2024-11-10T20:29:34,051 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8be33adc55ab4770839f9c195768a248, NAME => 'TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868 2024-11-10T20:29:34,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741836_1012 (size=64) 2024-11-10T20:29:34,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741836_1012 (size=64) 2024-11-10T20:29:34,057 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:29:34,057 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 8be33adc55ab4770839f9c195768a248, disabling compactions & flushes 2024-11-10T20:29:34,057 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:34,057 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:34,058 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. after waiting 0 ms 2024-11-10T20:29:34,058 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:34,058 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:34,058 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8be33adc55ab4770839f9c195768a248: Waiting for close lock at 1731270574057Disabling compacts and flushes for region at 1731270574057Disabling writes for close at 1731270574058 (+1 ms)Writing region close event to WAL at 1731270574058Closed at 1731270574058 2024-11-10T20:29:34,059 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T20:29:34,059 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731270574059"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731270574059"}]},"ts":"1731270574059"} 2024-11-10T20:29:34,062 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T20:29:34,063 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T20:29:34,063 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270574063"}]},"ts":"1731270574063"} 2024-11-10T20:29:34,065 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-11-10T20:29:34,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8be33adc55ab4770839f9c195768a248, ASSIGN}] 2024-11-10T20:29:34,067 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8be33adc55ab4770839f9c195768a248, ASSIGN 2024-11-10T20:29:34,068 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8be33adc55ab4770839f9c195768a248, ASSIGN; state=OFFLINE, location=734d44df782b,44413,1731270573213; forceNewPlan=false, retain=false 2024-11-10T20:29:34,218 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8be33adc55ab4770839f9c195768a248, regionState=OPENING, regionLocation=734d44df782b,44413,1731270573213 2024-11-10T20:29:34,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8be33adc55ab4770839f9c195768a248, ASSIGN because future has completed 2024-11-10T20:29:34,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213}] 2024-11-10T20:29:34,229 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:34,229 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:34,378 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:34,378 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8be33adc55ab4770839f9c195768a248, NAME => 'TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:29:34,379 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,379 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:29:34,379 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,379 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,380 INFO [StoreOpener-8be33adc55ab4770839f9c195768a248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,382 INFO [StoreOpener-8be33adc55ab4770839f9c195768a248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8be33adc55ab4770839f9c195768a248 columnFamilyName info 2024-11-10T20:29:34,382 DEBUG [StoreOpener-8be33adc55ab4770839f9c195768a248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:34,382 INFO [StoreOpener-8be33adc55ab4770839f9c195768a248-1 {}] regionserver.HStore(327): Store=8be33adc55ab4770839f9c195768a248/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:29:34,383 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,383 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,383 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,384 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,384 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,385 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,387 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:29:34,387 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8be33adc55ab4770839f9c195768a248; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=716930, jitterRate=-0.08837683498859406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T20:29:34,388 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:34,388 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8be33adc55ab4770839f9c195768a248: Running coprocessor pre-open hook at 1731270574379Writing region info on filesystem at 1731270574379Initializing all the Stores at 1731270574380 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270574380Cleaning up temporary data from old regions at 1731270574384 (+4 ms)Running coprocessor post-open hooks at 1731270574388 (+4 ms)Region opened successfully at 1731270574388 2024-11-10T20:29:34,389 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., pid=6, masterSystemTime=1731270574374 2024-11-10T20:29:34,391 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:34,391 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:34,392 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8be33adc55ab4770839f9c195768a248, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,44413,1731270573213 2024-11-10T20:29:34,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213 because future has completed 2024-11-10T20:29:34,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T20:29:34,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213 in 174 msec 2024-11-10T20:29:34,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T20:29:34,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8be33adc55ab4770839f9c195768a248, ASSIGN in 333 msec 2024-11-10T20:29:34,401 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T20:29:34,401 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731270574401"}]},"ts":"1731270574401"} 2024-11-10T20:29:34,403 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-11-10T20:29:34,404 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T20:29:34,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 365 msec 2024-11-10T20:29:35,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:35,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:36,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:36,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:37,007 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,007 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,007 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,007 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,008 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,008 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,008 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,008 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,023 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,023 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,023 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,024 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,024 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,024 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,027 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,027 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,027 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,029 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,231 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:37,231 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:37,533 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T20:29:37,534 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,535 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,535 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,535 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,535 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,535 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,536 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,536 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,553 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,554 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,554 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,554 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,554 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,555 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,558 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,559 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,559 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:37,561 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:29:38,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:38,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:39,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:39,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:39,454 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T20:29:39,454 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-11-10T20:29:40,233 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:40,233 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:41,234 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:41,234 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:42,234 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:42,234 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:42,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T20:29:42,804 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T20:29:42,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:29:42,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T20:29:42,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T20:29:42,805 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T20:29:42,806 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-10T20:29:42,806 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-10T20:29:43,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:43,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T20:29:44,083 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-11-10T20:29:44,083 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-11-10T20:29:44,085 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-11-10T20:29:44,085 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., hostname=734d44df782b,44413,1731270573213, seqNum=2] 2024-11-10T20:29:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:44,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8be33adc55ab4770839f9c195768a248 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:29:44,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/6d071c3f065a469bbba1702e7cde6c22 is 1080, key is row0001/info:/1731270584089/Put/seqid=0 2024-11-10T20:29:44,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741837_1013 (size=12509) 2024-11-10T20:29:44,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741837_1013 (size=12509) 2024-11-10T20:29:44,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/6d071c3f065a469bbba1702e7cde6c22 2024-11-10T20:29:44,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/6d071c3f065a469bbba1702e7cde6c22 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6d071c3f065a469bbba1702e7cde6c22 2024-11-10T20:29:44,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6d071c3f065a469bbba1702e7cde6c22, entries=7, sequenceid=11, filesize=12.2 K 2024-11-10T20:29:44,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T20:29:44,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 8be33adc55ab4770839f9c195768a248 in 40ms, sequenceid=11, compaction requested=false 2024-11-10T20:29:44,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8be33adc55ab4770839f9c195768a248: 2024-11-10T20:29:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:57004 deadline: 1731270594138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213 2024-11-10T20:29:44,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., hostname=734d44df782b,44413,1731270573213, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., hostname=734d44df782b,44413,1731270573213, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T20:29:44,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., hostname=734d44df782b,44413,1731270573213, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T20:29:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., hostname=734d44df782b,44413,1731270573213, seqNum=2 because the exception is null or not the one we care about 2024-11-10T20:29:44,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:44,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:45,236 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:45,236 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:46,236 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:46,236 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:47,237 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:47,237 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:48,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:48,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:49,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:49,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:50,239 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:50,239 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:51,239 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:51,239 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:52,240 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:52,240 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:53,241 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:53,241 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:54,241 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:54,242 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:54,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8be33adc55ab4770839f9c195768a248 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-11-10T20:29:54,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/9330157011df4769bdb5d9a7c1eff0a3 is 1080, key is row0008/info:/1731270584100/Put/seqid=0 2024-11-10T20:29:54,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741838_1014 (size=29761) 2024-11-10T20:29:54,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741838_1014 (size=29761) 2024-11-10T20:29:54,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/9330157011df4769bdb5d9a7c1eff0a3 2024-11-10T20:29:54,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/9330157011df4769bdb5d9a7c1eff0a3 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/9330157011df4769bdb5d9a7c1eff0a3 2024-11-10T20:29:54,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/9330157011df4769bdb5d9a7c1eff0a3, entries=23, sequenceid=37, filesize=29.1 K 2024-11-10T20:29:54,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 8be33adc55ab4770839f9c195768a248 in 25ms, sequenceid=37, compaction requested=false 2024-11-10T20:29:54,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8be33adc55ab4770839f9c195768a248: 2024-11-10T20:29:54,278 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K 2024-11-10T20:29:54,278 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:54,278 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/9330157011df4769bdb5d9a7c1eff0a3 because midkey is the same as first or last row 2024-11-10T20:29:55,242 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:55,242 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:56,242 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:56,242 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:56,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8be33adc55ab4770839f9c195768a248 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:29:56,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/6ed04b34c26c41abbbe9347dab35c12b is 1080, key is row0031/info:/1731270594254/Put/seqid=0 2024-11-10T20:29:56,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741839_1015 (size=12509) 2024-11-10T20:29:56,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741839_1015 (size=12509) 2024-11-10T20:29:56,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/6ed04b34c26c41abbbe9347dab35c12b 2024-11-10T20:29:56,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/6ed04b34c26c41abbbe9347dab35c12b as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6ed04b34c26c41abbbe9347dab35c12b 2024-11-10T20:29:56,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6ed04b34c26c41abbbe9347dab35c12b, entries=7, sequenceid=47, filesize=12.2 K 2024-11-10T20:29:56,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 8be33adc55ab4770839f9c195768a248 in 23ms, sequenceid=47, compaction requested=true 2024-11-10T20:29:56,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8be33adc55ab4770839f9c195768a248: 2024-11-10T20:29:56,290 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K 2024-11-10T20:29:56,290 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:56,290 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/9330157011df4769bdb5d9a7c1eff0a3 because midkey is the same as first or last row 2024-11-10T20:29:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:56,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8be33adc55ab4770839f9c195768a248:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:29:56,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:56,290 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:29:56,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8be33adc55ab4770839f9c195768a248 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T20:29:56,291 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:29:56,291 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1541): 8be33adc55ab4770839f9c195768a248/info is initiating minor compaction (all files) 2024-11-10T20:29:56,292 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8be33adc55ab4770839f9c195768a248/info in TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:56,292 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6d071c3f065a469bbba1702e7cde6c22, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/9330157011df4769bdb5d9a7c1eff0a3, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6ed04b34c26c41abbbe9347dab35c12b] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp, totalSize=53.5 K 2024-11-10T20:29:56,293 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d071c3f065a469bbba1702e7cde6c22, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731270584089 2024-11-10T20:29:56,293 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9330157011df4769bdb5d9a7c1eff0a3, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731270584100 2024-11-10T20:29:56,294 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ed04b34c26c41abbbe9347dab35c12b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731270594254 2024-11-10T20:29:56,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/b12282d589294a1e91f3b88786807daa is 1080, key is row0038/info:/1731270596267/Put/seqid=0 2024-11-10T20:29:56,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741840_1016 (size=21141) 2024-11-10T20:29:56,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741840_1016 (size=21141) 2024-11-10T20:29:56,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/b12282d589294a1e91f3b88786807daa 2024-11-10T20:29:56,308 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8be33adc55ab4770839f9c195768a248#info#compaction#58 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:29:56,309 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/b6205ef80fd1494f8b66e6d7c2f60470 is 1080, key is row0001/info:/1731270584089/Put/seqid=0 2024-11-10T20:29:56,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/b12282d589294a1e91f3b88786807daa as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b12282d589294a1e91f3b88786807daa 2024-11-10T20:29:56,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b12282d589294a1e91f3b88786807daa, entries=15, sequenceid=65, filesize=20.6 K 2024-11-10T20:29:56,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741841_1017 (size=44978) 2024-11-10T20:29:56,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741841_1017 (size=44978) 2024-11-10T20:29:56,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=12.61 KB/12912 for 8be33adc55ab4770839f9c195768a248 in 27ms, sequenceid=65, compaction requested=false 2024-11-10T20:29:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8be33adc55ab4770839f9c195768a248: 2024-11-10T20:29:56,318 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=74.1 K, sizeToCheck=16.0 K 2024-11-10T20:29:56,318 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:56,318 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/9330157011df4769bdb5d9a7c1eff0a3 because midkey is the same as first or last row 2024-11-10T20:29:56,323 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/b6205ef80fd1494f8b66e6d7c2f60470 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470 2024-11-10T20:29:56,329 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8be33adc55ab4770839f9c195768a248/info of 8be33adc55ab4770839f9c195768a248 into b6205ef80fd1494f8b66e6d7c2f60470(size=43.9 K), total size for store is 64.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8be33adc55ab4770839f9c195768a248: 2024-11-10T20:29:56,330 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., storeName=8be33adc55ab4770839f9c195768a248/info, priority=13, startTime=1731270596290; duration=0sec 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470 because midkey is the same as first or last row 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470 because midkey is the same as first or last row 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470 because midkey is the same as first or last row 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:56,330 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8be33adc55ab4770839f9c195768a248:info 2024-11-10T20:29:57,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:57,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:58,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:58,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8be33adc55ab4770839f9c195768a248 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-11-10T20:29:58,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/c4d7dff9715a47d18b879cc45d1168c3 is 1080, key is row0053/info:/1731270596291/Put/seqid=0 2024-11-10T20:29:58,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741842_1018 (size=18987) 2024-11-10T20:29:58,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741842_1018 (size=18987) 2024-11-10T20:29:58,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/c4d7dff9715a47d18b879cc45d1168c3 2024-11-10T20:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/c4d7dff9715a47d18b879cc45d1168c3 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/c4d7dff9715a47d18b879cc45d1168c3 2024-11-10T20:29:58,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/c4d7dff9715a47d18b879cc45d1168c3, entries=13, sequenceid=82, filesize=18.5 K 2024-11-10T20:29:58,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=13.66 KB/13988 for 8be33adc55ab4770839f9c195768a248 in 24ms, sequenceid=82, compaction requested=true 2024-11-10T20:29:58,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8be33adc55ab4770839f9c195768a248: 2024-11-10T20:29:58,339 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=83.1 K, sizeToCheck=16.0 K 2024-11-10T20:29:58,339 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:58,339 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470 because midkey is the same as first or last row 2024-11-10T20:29:58,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8be33adc55ab4770839f9c195768a248:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:29:58,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:58,340 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:29:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8be33adc55ab4770839f9c195768a248 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T20:29:58,341 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:29:58,341 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1541): 8be33adc55ab4770839f9c195768a248/info is initiating minor compaction (all files) 2024-11-10T20:29:58,341 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8be33adc55ab4770839f9c195768a248/info in TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:58,341 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b12282d589294a1e91f3b88786807daa, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/c4d7dff9715a47d18b879cc45d1168c3] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp, totalSize=83.1 K 2024-11-10T20:29:58,341 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6205ef80fd1494f8b66e6d7c2f60470, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731270584089 2024-11-10T20:29:58,342 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12282d589294a1e91f3b88786807daa, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731270596267 2024-11-10T20:29:58,342 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4d7dff9715a47d18b879cc45d1168c3, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731270596291 2024-11-10T20:29:58,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/f9889582fdb94b3cbe19f422990db53e is 1080, key is row0066/info:/1731270598317/Put/seqid=0 2024-11-10T20:29:58,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741843_1019 (size=20064) 2024-11-10T20:29:58,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741843_1019 (size=20064) 2024-11-10T20:29:58,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/f9889582fdb94b3cbe19f422990db53e 2024-11-10T20:29:58,356 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8be33adc55ab4770839f9c195768a248#info#compaction#61 average throughput is 33.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:29:58,357 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/46e22e59335046bebad71837e99ec257 is 1080, key is row0001/info:/1731270584089/Put/seqid=0 2024-11-10T20:29:58,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/f9889582fdb94b3cbe19f422990db53e as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/f9889582fdb94b3cbe19f422990db53e 2024-11-10T20:29:58,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741844_1020 (size=75378) 2024-11-10T20:29:58,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741844_1020 (size=75378) 2024-11-10T20:29:58,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/f9889582fdb94b3cbe19f422990db53e, entries=14, sequenceid=99, filesize=19.6 K 2024-11-10T20:29:58,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=13.66 KB/13988 for 8be33adc55ab4770839f9c195768a248 in 27ms, sequenceid=99, compaction requested=false 2024-11-10T20:29:58,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8be33adc55ab4770839f9c195768a248: 2024-11-10T20:29:58,367 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.7 K, sizeToCheck=16.0 K 2024-11-10T20:29:58,367 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:58,367 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470 because midkey is the same as first or last row 2024-11-10T20:29:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8be33adc55ab4770839f9c195768a248 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T20:29:58,369 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/46e22e59335046bebad71837e99ec257 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/46e22e59335046bebad71837e99ec257 2024-11-10T20:29:58,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/acfd7de9e7e34a6ba6161b50cc3710eb is 1080, key is row0080/info:/1731270598341/Put/seqid=0 2024-11-10T20:29:58,376 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8be33adc55ab4770839f9c195768a248/info of 8be33adc55ab4770839f9c195768a248 into 46e22e59335046bebad71837e99ec257(size=73.6 K), total size for store is 93.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:29:58,376 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8be33adc55ab4770839f9c195768a248: 2024-11-10T20:29:58,376 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., storeName=8be33adc55ab4770839f9c195768a248/info, priority=13, startTime=1731270598339; duration=0sec 2024-11-10T20:29:58,376 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K 2024-11-10T20:29:58,376 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:58,376 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K 2024-11-10T20:29:58,376 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:58,377 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K 2024-11-10T20:29:58,377 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:58,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741845_1021 (size=20064) 2024-11-10T20:29:58,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741845_1021 (size=20064) 2024-11-10T20:29:58,379 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:58,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/acfd7de9e7e34a6ba6161b50cc3710eb 2024-11-10T20:29:58,379 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:58,379 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8be33adc55ab4770839f9c195768a248:info 2024-11-10T20:29:58,380 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35643 {}] assignment.AssignmentManager(1363): Split request from 734d44df782b,44413,1731270573213, parent={ENCODED => 8be33adc55ab4770839f9c195768a248, NAME => 'TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-11-10T20:29:58,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/acfd7de9e7e34a6ba6161b50cc3710eb as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/acfd7de9e7e34a6ba6161b50cc3710eb 2024-11-10T20:29:58,385 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35643 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=734d44df782b,44413,1731270573213 2024-11-10T20:29:58,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/acfd7de9e7e34a6ba6161b50cc3710eb, entries=14, sequenceid=116, filesize=19.6 K 2024-11-10T20:29:58,389 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35643 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=dc40cbb5ebd30c95e94857ec7ee91b25, daughterB=08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:58,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=3.15 KB/3228 for 8be33adc55ab4770839f9c195768a248 in 23ms, sequenceid=116, compaction requested=true 2024-11-10T20:29:58,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8be33adc55ab4770839f9c195768a248: 2024-11-10T20:29:58,390 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K 2024-11-10T20:29:58,391 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:58,391 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=dc40cbb5ebd30c95e94857ec7ee91b25, daughterB=08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:58,391 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K 2024-11-10T20:29:58,391 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=dc40cbb5ebd30c95e94857ec7ee91b25, daughterB=08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:58,391 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:58,391 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=dc40cbb5ebd30c95e94857ec7ee91b25, daughterB=08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:58,391 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K 2024-11-10T20:29:58,391 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T20:29:58,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:58,392 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35643 {}] assignment.AssignmentManager(1363): Split request from 734d44df782b,44413,1731270573213, parent={ENCODED => 8be33adc55ab4770839f9c195768a248, NAME => 'TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-11-10T20:29:58,393 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35643 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=SPLITTING, location=734d44df782b,44413,1731270573213 2024-11-10T20:29:58,394 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35643 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=6e8cc42e4509b73375acb4eb50e84010, daughterB=6ddb6304aa0445ea37e99d14aedf1097 2024-11-10T20:29:58,394 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(842): Waiting on xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=6e8cc42e4509b73375acb4eb50e84010, daughterB=6ddb6304aa0445ea37e99d14aedf1097 held by pid=7 2024-11-10T20:29:58,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8be33adc55ab4770839f9c195768a248, UNASSIGN}] 2024-11-10T20:29:58,401 DEBUG [PEWorker-3 {}] assignment.SplitTableRegionProcedure(162): LOCK_EVENT_WAIT SchemaLocking[serverLocks={},namespaceLocks={hbase=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},tableLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},regionLocks={},peerLocks={},metaLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},globalLocks={}] 2024-11-10T20:29:58,401 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8be33adc55ab4770839f9c195768a248, UNASSIGN 2024-11-10T20:29:58,401 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=6e8cc42e4509b73375acb4eb50e84010, daughterB=6ddb6304aa0445ea37e99d14aedf1097 2024-11-10T20:29:58,403 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8be33adc55ab4770839f9c195768a248, regionState=CLOSING, regionLocation=734d44df782b,44413,1731270573213 2024-11-10T20:29:58,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8be33adc55ab4770839f9c195768a248, UNASSIGN because future has completed 2024-11-10T20:29:58,406 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-11-10T20:29:58,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213}] 2024-11-10T20:29:58,563 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,564 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-11-10T20:29:58,564 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 8be33adc55ab4770839f9c195768a248, disabling compactions & flushes 2024-11-10T20:29:58,564 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:58,564 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:58,564 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. after waiting 0 ms 2024-11-10T20:29:58,564 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:58,565 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(2902): Flushing 8be33adc55ab4770839f9c195768a248 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-11-10T20:29:58,569 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/ad9617199a414f8c95d21c7777eb772c is 1080, key is row0094/info:/1731270598369/Put/seqid=0 2024-11-10T20:29:58,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741846_1022 (size=8193) 2024-11-10T20:29:58,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741846_1022 (size=8193) 2024-11-10T20:29:58,574 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/ad9617199a414f8c95d21c7777eb772c 2024-11-10T20:29:58,579 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/.tmp/info/ad9617199a414f8c95d21c7777eb772c as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/ad9617199a414f8c95d21c7777eb772c 2024-11-10T20:29:58,584 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/ad9617199a414f8c95d21c7777eb772c, entries=3, sequenceid=123, filesize=8.0 K 2024-11-10T20:29:58,585 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 8be33adc55ab4770839f9c195768a248 in 21ms, sequenceid=123, compaction requested=true 2024-11-10T20:29:58,586 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6d071c3f065a469bbba1702e7cde6c22, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/9330157011df4769bdb5d9a7c1eff0a3, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6ed04b34c26c41abbbe9347dab35c12b, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b12282d589294a1e91f3b88786807daa, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/c4d7dff9715a47d18b879cc45d1168c3] to archive 2024-11-10T20:29:58,587 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T20:29:58,589 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6d071c3f065a469bbba1702e7cde6c22 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6d071c3f065a469bbba1702e7cde6c22 2024-11-10T20:29:58,590 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/9330157011df4769bdb5d9a7c1eff0a3 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/9330157011df4769bdb5d9a7c1eff0a3 2024-11-10T20:29:58,591 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b6205ef80fd1494f8b66e6d7c2f60470 2024-11-10T20:29:58,592 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6ed04b34c26c41abbbe9347dab35c12b to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/6ed04b34c26c41abbbe9347dab35c12b 2024-11-10T20:29:58,593 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b12282d589294a1e91f3b88786807daa to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/b12282d589294a1e91f3b88786807daa 2024-11-10T20:29:58,594 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/c4d7dff9715a47d18b879cc45d1168c3 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/c4d7dff9715a47d18b879cc45d1168c3 2024-11-10T20:29:58,600 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=1 2024-11-10T20:29:58,600 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. 2024-11-10T20:29:58,600 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 8be33adc55ab4770839f9c195768a248: Waiting for close lock at 1731270598564Running coprocessor pre-close hooks at 1731270598564Disabling compacts and flushes for region at 1731270598564Disabling writes for close at 1731270598564Obtaining lock to block concurrent updates at 1731270598565 (+1 ms)Preparing flush snapshotting stores in 8be33adc55ab4770839f9c195768a248 at 1731270598565Finished memstore snapshotting TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731270598565Flushing stores of TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. at 1731270598566 (+1 ms)Flushing 8be33adc55ab4770839f9c195768a248/info: creating writer at 1731270598566Flushing 8be33adc55ab4770839f9c195768a248/info: appending metadata at 1731270598568 (+2 ms)Flushing 8be33adc55ab4770839f9c195768a248/info: closing flushed file at 1731270598568Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b6f3bd6: reopening flushed file at 1731270598579 (+11 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 8be33adc55ab4770839f9c195768a248 in 21ms, sequenceid=123, compaction requested=true at 1731270598585 (+6 ms)Writing region close event to WAL at 1731270598597 (+12 ms)Running coprocessor post-close hooks at 1731270598600 (+3 ms)Closed at 1731270598600 2024-11-10T20:29:58,603 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,603 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8be33adc55ab4770839f9c195768a248, regionState=CLOSED 2024-11-10T20:29:58,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213 because future has completed 2024-11-10T20:29:58,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T20:29:58,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 8be33adc55ab4770839f9c195768a248, server=734d44df782b,44413,1731270573213 in 200 msec 2024-11-10T20:29:58,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-10T20:29:58,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8be33adc55ab4770839f9c195768a248, UNASSIGN in 210 msec 2024-11-10T20:29:58,618 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:58,621 INFO [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 4 storefiles, region=8be33adc55ab4770839f9c195768a248, threads=4 2024-11-10T20:29:58,624 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/46e22e59335046bebad71837e99ec257 for region: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,624 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/acfd7de9e7e34a6ba6161b50cc3710eb for region: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,624 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/ad9617199a414f8c95d21c7777eb772c for region: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,624 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/f9889582fdb94b3cbe19f422990db53e for region: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,633 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/acfd7de9e7e34a6ba6161b50cc3710eb, top=true 2024-11-10T20:29:58,634 DEBUG [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/f9889582fdb94b3cbe19f422990db53e, top=true 2024-11-10T20:29:58,634 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/ad9617199a414f8c95d21c7777eb772c, top=true 2024-11-10T20:29:58,639 INFO [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-acfd7de9e7e34a6ba6161b50cc3710eb for child: 08af9de3a5999d7a5aa81e049047fc83, parent: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,639 INFO [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-f9889582fdb94b3cbe19f422990db53e for child: 08af9de3a5999d7a5aa81e049047fc83, parent: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,639 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/acfd7de9e7e34a6ba6161b50cc3710eb for region: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,639 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/f9889582fdb94b3cbe19f422990db53e for region: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,639 INFO [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-ad9617199a414f8c95d21c7777eb772c for child: 08af9de3a5999d7a5aa81e049047fc83, parent: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,639 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/ad9617199a414f8c95d21c7777eb772c for region: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741847_1023 (size=27) 2024-11-10T20:29:58,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741847_1023 (size=27) 2024-11-10T20:29:58,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741848_1024 (size=27) 2024-11-10T20:29:58,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741848_1024 (size=27) 2024-11-10T20:29:58,652 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/46e22e59335046bebad71837e99ec257 for region: 8be33adc55ab4770839f9c195768a248 2024-11-10T20:29:58,655 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 8be33adc55ab4770839f9c195768a248 Daughter A: [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248] storefiles, Daughter B: [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-acfd7de9e7e34a6ba6161b50cc3710eb, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-ad9617199a414f8c95d21c7777eb772c, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-f9889582fdb94b3cbe19f422990db53e] storefiles. 2024-11-10T20:29:58,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741849_1025 (size=71) 2024-11-10T20:29:58,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741849_1025 (size=71) 2024-11-10T20:29:58,666 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:58,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741850_1026 (size=71) 2024-11-10T20:29:58,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741850_1026 (size=71) 2024-11-10T20:29:58,680 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:58,690 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1 2024-11-10T20:29:58,692 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1 2024-11-10T20:29:58,694 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731270598694"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731270598694"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731270598694"}]},"ts":"1731270598694"} 2024-11-10T20:29:58,694 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731270598694"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731270598694"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731270598694"}]},"ts":"1731270598694"} 2024-11-10T20:29:58,694 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731270598694"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731270598694"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731270598694"}]},"ts":"1731270598694"} 2024-11-10T20:29:58,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=dc40cbb5ebd30c95e94857ec7ee91b25, ASSIGN}, {pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=08af9de3a5999d7a5aa81e049047fc83, ASSIGN}] 2024-11-10T20:29:58,714 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=dc40cbb5ebd30c95e94857ec7ee91b25, ASSIGN 2024-11-10T20:29:58,714 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=08af9de3a5999d7a5aa81e049047fc83, ASSIGN 2024-11-10T20:29:58,714 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=dc40cbb5ebd30c95e94857ec7ee91b25, ASSIGN; state=SPLITTING_NEW, location=734d44df782b,44413,1731270573213; forceNewPlan=false, retain=false 2024-11-10T20:29:58,714 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=08af9de3a5999d7a5aa81e049047fc83, ASSIGN; state=SPLITTING_NEW, location=734d44df782b,44413,1731270573213; forceNewPlan=false, retain=false 2024-11-10T20:29:58,865 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=08af9de3a5999d7a5aa81e049047fc83, regionState=OPENING, regionLocation=734d44df782b,44413,1731270573213 2024-11-10T20:29:58,865 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=dc40cbb5ebd30c95e94857ec7ee91b25, regionState=OPENING, regionLocation=734d44df782b,44413,1731270573213 2024-11-10T20:29:58,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=08af9de3a5999d7a5aa81e049047fc83, ASSIGN because future has completed 2024-11-10T20:29:58,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08af9de3a5999d7a5aa81e049047fc83, server=734d44df782b,44413,1731270573213}] 2024-11-10T20:29:58,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=dc40cbb5ebd30c95e94857ec7ee91b25, ASSIGN because future has completed 2024-11-10T20:29:58,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc40cbb5ebd30c95e94857ec7ee91b25, server=734d44df782b,44413,1731270573213}] 2024-11-10T20:29:59,024 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:29:59,024 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 08af9de3a5999d7a5aa81e049047fc83, NAME => 'TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.', STARTKEY => 'row0062', ENDKEY => ''} 2024-11-10T20:29:59,025 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,025 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:29:59,025 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,025 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,026 INFO [StoreOpener-08af9de3a5999d7a5aa81e049047fc83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,027 INFO [StoreOpener-08af9de3a5999d7a5aa81e049047fc83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08af9de3a5999d7a5aa81e049047fc83 columnFamilyName info 2024-11-10T20:29:59,027 DEBUG [StoreOpener-08af9de3a5999d7a5aa81e049047fc83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:59,037 DEBUG [StoreOpener-08af9de3a5999d7a5aa81e049047fc83-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248->hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/46e22e59335046bebad71837e99ec257-top 2024-11-10T20:29:59,041 DEBUG [StoreOpener-08af9de3a5999d7a5aa81e049047fc83-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-acfd7de9e7e34a6ba6161b50cc3710eb 2024-11-10T20:29:59,045 DEBUG [StoreOpener-08af9de3a5999d7a5aa81e049047fc83-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-ad9617199a414f8c95d21c7777eb772c 2024-11-10T20:29:59,049 DEBUG [StoreOpener-08af9de3a5999d7a5aa81e049047fc83-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-f9889582fdb94b3cbe19f422990db53e 2024-11-10T20:29:59,049 INFO [StoreOpener-08af9de3a5999d7a5aa81e049047fc83-1 {}] regionserver.HStore(327): Store=08af9de3a5999d7a5aa81e049047fc83/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:29:59,049 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,050 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,051 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,051 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,051 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,053 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,054 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 08af9de3a5999d7a5aa81e049047fc83; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=883598, jitterRate=0.12355338037014008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T20:29:59,054 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:29:59,054 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 08af9de3a5999d7a5aa81e049047fc83: Running coprocessor pre-open hook at 1731270599025Writing region info on filesystem at 1731270599025Initializing all the Stores at 1731270599025Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270599026 (+1 ms)Cleaning up temporary data from old regions at 1731270599051 (+25 ms)Running coprocessor post-open hooks at 1731270599054 (+3 ms)Region opened successfully at 1731270599054 2024-11-10T20:29:59,055 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., pid=13, masterSystemTime=1731270599021 2024-11-10T20:29:59,055 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 08af9de3a5999d7a5aa81e049047fc83:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:29:59,055 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:59,055 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T20:29:59,057 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:29:59,057 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1541): 08af9de3a5999d7a5aa81e049047fc83/info is initiating minor compaction (all files) 2024-11-10T20:29:59,057 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08af9de3a5999d7a5aa81e049047fc83/info in TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:29:59,057 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:29:59,057 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248->hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/46e22e59335046bebad71837e99ec257-top, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-f9889582fdb94b3cbe19f422990db53e, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-acfd7de9e7e34a6ba6161b50cc3710eb, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-ad9617199a414f8c95d21c7777eb772c] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp, totalSize=120.8 K 2024-11-10T20:29:59,057 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:29:59,058 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:29:59,058 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7752): Opening region: {ENCODED => dc40cbb5ebd30c95e94857ec7ee91b25, NAME => 'TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25.', STARTKEY => '', ENDKEY => 'row0062'} 2024-11-10T20:29:59,058 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,058 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:29:59,058 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7794): checking encryption for dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,058 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7797): checking classloading for dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,058 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731270584089 2024-11-10T20:29:59,058 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=08af9de3a5999d7a5aa81e049047fc83, regionState=OPEN, openSeqNum=127, regionLocation=734d44df782b,44413,1731270573213 2024-11-10T20:29:59,059 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-f9889582fdb94b3cbe19f422990db53e, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731270598317 2024-11-10T20:29:59,059 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-acfd7de9e7e34a6ba6161b50cc3710eb, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731270598341 2024-11-10T20:29:59,059 INFO [StoreOpener-dc40cbb5ebd30c95e94857ec7ee91b25-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,059 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-ad9617199a414f8c95d21c7777eb772c, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731270598369 2024-11-10T20:29:59,060 INFO [StoreOpener-dc40cbb5ebd30c95e94857ec7ee91b25-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc40cbb5ebd30c95e94857ec7ee91b25 columnFamilyName info 2024-11-10T20:29:59,060 DEBUG [StoreOpener-dc40cbb5ebd30c95e94857ec7ee91b25-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:29:59,060 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-11-10T20:29:59,060 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T20:29:59,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB 2024-11-10T20:29:59,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08af9de3a5999d7a5aa81e049047fc83, server=734d44df782b,44413,1731270573213 because future has completed 2024-11-10T20:29:59,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-10T20:29:59,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 08af9de3a5999d7a5aa81e049047fc83, server=734d44df782b,44413,1731270573213 in 194 msec 2024-11-10T20:29:59,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=08af9de3a5999d7a5aa81e049047fc83, ASSIGN in 353 msec 2024-11-10T20:29:59,078 DEBUG [StoreOpener-dc40cbb5ebd30c95e94857ec7ee91b25-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248->hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/46e22e59335046bebad71837e99ec257-bottom 2024-11-10T20:29:59,078 INFO [StoreOpener-dc40cbb5ebd30c95e94857ec7ee91b25-1 {}] regionserver.HStore(327): Store=dc40cbb5ebd30c95e94857ec7ee91b25/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:29:59,078 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1038): replaying wal for dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,079 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,081 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/info/a6eddda0727d4141a4751daa45ffbcda is 193, key is TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83./info:regioninfo/1731270599058/Put/seqid=0 2024-11-10T20:29:59,081 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1048): stopping wal replay for dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,081 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1060): Cleaning up temporary data for dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,083 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1093): writing seq id for dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,084 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1114): Opened dc40cbb5ebd30c95e94857ec7ee91b25; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=776051, jitterRate=-0.013200968503952026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T20:29:59,084 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:29:59,084 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1006): Region open journal for dc40cbb5ebd30c95e94857ec7ee91b25: Running coprocessor pre-open hook at 1731270599058Writing region info on filesystem at 1731270599058Initializing all the Stores at 1731270599059 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270599059Cleaning up temporary data from old regions at 1731270599081 (+22 ms)Running coprocessor post-open hooks at 1731270599084 (+3 ms)Region opened successfully at 1731270599084 2024-11-10T20:29:59,085 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25., pid=14, masterSystemTime=1731270599021 2024-11-10T20:29:59,086 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(403): Add compact mark for store dc40cbb5ebd30c95e94857ec7ee91b25:info, priority=-2147483648, current under compaction store size is 2 2024-11-10T20:29:59,086 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:59,086 DEBUG [RS:0;734d44df782b:44413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-10T20:29:59,086 INFO [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:29:59,086 DEBUG [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.HStore(1541): dc40cbb5ebd30c95e94857ec7ee91b25/info is initiating minor compaction (all files) 2024-11-10T20:29:59,086 INFO [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc40cbb5ebd30c95e94857ec7ee91b25/info in TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:29:59,087 INFO [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248->hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/46e22e59335046bebad71837e99ec257-bottom] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/.tmp, totalSize=73.6 K 2024-11-10T20:29:59,087 DEBUG [RS:0;734d44df782b:44413-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731270584089 2024-11-10T20:29:59,088 DEBUG [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:29:59,088 INFO [RS_OPEN_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:29:59,089 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=dc40cbb5ebd30c95e94857ec7ee91b25, regionState=OPEN, openSeqNum=127, regionLocation=734d44df782b,44413,1731270573213 2024-11-10T20:29:59,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc40cbb5ebd30c95e94857ec7ee91b25, server=734d44df782b,44413,1731270573213 because future has completed 2024-11-10T20:29:59,095 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08af9de3a5999d7a5aa81e049047fc83#info#compaction#65 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:29:59,095 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/657561f87134480f95db946a4e5dff8e is 1080, key is row0062/info:/1731270596307/Put/seqid=0 2024-11-10T20:29:59,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741851_1027 (size=9882) 2024-11-10T20:29:59,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741851_1027 (size=9882) 2024-11-10T20:29:59,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/info/a6eddda0727d4141a4751daa45ffbcda 2024-11-10T20:29:59,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=11 2024-11-10T20:29:59,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure dc40cbb5ebd30c95e94857ec7ee91b25, server=734d44df782b,44413,1731270573213 in 228 msec 2024-11-10T20:29:59,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7 2024-11-10T20:29:59,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=dc40cbb5ebd30c95e94857ec7ee91b25, ASSIGN in 389 msec 2024-11-10T20:29:59,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=dc40cbb5ebd30c95e94857ec7ee91b25, daughterB=08af9de3a5999d7a5aa81e049047fc83 in 717 msec 2024-11-10T20:29:59,105 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=6e8cc42e4509b73375acb4eb50e84010, daughterB=6ddb6304aa0445ea37e99d14aedf1097 2024-11-10T20:29:59,105 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=6e8cc42e4509b73375acb4eb50e84010, daughterB=6ddb6304aa0445ea37e99d14aedf1097 2024-11-10T20:29:59,105 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=6e8cc42e4509b73375acb4eb50e84010, daughterB=6ddb6304aa0445ea37e99d14aedf1097 2024-11-10T20:29:59,106 INFO [PEWorker-2 {}] assignment.SplitTableRegionProcedure(534): Split of {ENCODED => 8be33adc55ab4770839f9c195768a248, NAME => 'TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248.', STARTKEY => '', ENDKEY => ''} skipped; state is already SPLIT 2024-11-10T20:29:59,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8be33adc55ab4770839f9c195768a248, daughterA=6e8cc42e4509b73375acb4eb50e84010, daughterB=6ddb6304aa0445ea37e99d14aedf1097 in 713 msec 2024-11-10T20:29:59,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741852_1028 (size=43081) 2024-11-10T20:29:59,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741852_1028 (size=43081) 2024-11-10T20:29:59,116 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/657561f87134480f95db946a4e5dff8e as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/657561f87134480f95db946a4e5dff8e 2024-11-10T20:29:59,116 INFO [RS:0;734d44df782b:44413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc40cbb5ebd30c95e94857ec7ee91b25#info#compaction#66 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:29:59,117 DEBUG [RS:0;734d44df782b:44413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/.tmp/info/3e5cbcc6a85448228e829e4dfcde46f7 is 1080, key is row0001/info:/1731270584089/Put/seqid=0 2024-11-10T20:29:59,123 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 08af9de3a5999d7a5aa81e049047fc83/info of 08af9de3a5999d7a5aa81e049047fc83 into 657561f87134480f95db946a4e5dff8e(size=42.1 K), total size for store is 42.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:29:59,123 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:29:59,123 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., storeName=08af9de3a5999d7a5aa81e049047fc83/info, priority=12, startTime=1731270599055; duration=0sec 2024-11-10T20:29:59,123 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:59,123 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08af9de3a5999d7a5aa81e049047fc83:info 2024-11-10T20:29:59,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741853_1029 (size=70862) 2024-11-10T20:29:59,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741853_1029 (size=70862) 2024-11-10T20:29:59,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/ns/6527be7170a049588fe38a2a22142d9a is 43, key is default/ns:d/1731270573998/Put/seqid=0 2024-11-10T20:29:59,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741854_1030 (size=5153) 2024-11-10T20:29:59,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741854_1030 (size=5153) 2024-11-10T20:29:59,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/ns/6527be7170a049588fe38a2a22142d9a 2024-11-10T20:29:59,133 DEBUG [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/.tmp/info/3e5cbcc6a85448228e829e4dfcde46f7 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/info/3e5cbcc6a85448228e829e4dfcde46f7 2024-11-10T20:29:59,140 INFO [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in dc40cbb5ebd30c95e94857ec7ee91b25/info of dc40cbb5ebd30c95e94857ec7ee91b25 into 3e5cbcc6a85448228e829e4dfcde46f7(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:29:59,140 DEBUG [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc40cbb5ebd30c95e94857ec7ee91b25: 2024-11-10T20:29:59,140 INFO [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25., storeName=dc40cbb5ebd30c95e94857ec7ee91b25/info, priority=15, startTime=1731270599085; duration=0sec 2024-11-10T20:29:59,140 DEBUG [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:29:59,140 DEBUG [RS:0;734d44df782b:44413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc40cbb5ebd30c95e94857ec7ee91b25:info 2024-11-10T20:29:59,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/table/a67c50e1436a4080842a887c288b0029 is 65, key is TestLogRolling-testLogRolling/table:state/1731270574401/Put/seqid=0 2024-11-10T20:29:59,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741855_1031 (size=5340) 2024-11-10T20:29:59,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741855_1031 (size=5340) 2024-11-10T20:29:59,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/table/a67c50e1436a4080842a887c288b0029 2024-11-10T20:29:59,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/info/a6eddda0727d4141a4751daa45ffbcda as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/info/a6eddda0727d4141a4751daa45ffbcda 2024-11-10T20:29:59,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/info/a6eddda0727d4141a4751daa45ffbcda, entries=30, sequenceid=17, filesize=9.7 K 2024-11-10T20:29:59,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/ns/6527be7170a049588fe38a2a22142d9a as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/ns/6527be7170a049588fe38a2a22142d9a 2024-11-10T20:29:59,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/ns/6527be7170a049588fe38a2a22142d9a, entries=2, sequenceid=17, filesize=5.0 K 2024-11-10T20:29:59,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/table/a67c50e1436a4080842a887c288b0029 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/table/a67c50e1436a4080842a887c288b0029 2024-11-10T20:29:59,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/table/a67c50e1436a4080842a887c288b0029, entries=2, sequenceid=17, filesize=5.2 K 2024-11-10T20:29:59,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 123ms, sequenceid=17, compaction requested=false 2024-11-10T20:29:59,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-10T20:29:59,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:29:59,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:00,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:00,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:57004 deadline: 1731270610374, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. is not online on 734d44df782b,44413,1731270573213 2024-11-10T20:30:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., hostname=734d44df782b,44413,1731270573213, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., hostname=734d44df782b,44413,1731270573213, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. is not online on 734d44df782b,44413,1731270573213 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T20:30:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., hostname=734d44df782b,44413,1731270573213, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248. is not online on 734d44df782b,44413,1731270573213 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T20:30:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731270574039.8be33adc55ab4770839f9c195768a248., hostname=734d44df782b,44413,1731270573213, seqNum=2 from cache 2024-11-10T20:30:01,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:01,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:02,246 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:02,246 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:03,149 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T20:30:03,247 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:03,247 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:03,601 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,601 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,602 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,602 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,602 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,602 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,603 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,603 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,620 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,620 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,620 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,620 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,620 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,621 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,624 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,624 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,624 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:03,626 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,133 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T20:30:04,134 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,134 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,135 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,135 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,135 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,135 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,136 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,136 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,155 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,155 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,156 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,156 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,156 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,157 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,160 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,161 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,161 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,163 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T20:30:04,247 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:04,247 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:05,248 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:05,248 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:06,248 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:06,248 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:07,249 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:07,249 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:08,250 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:08,250 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:09,250 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:09,250 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:10,251 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:10,251 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., hostname=734d44df782b,44413,1731270573213, seqNum=127] 2024-11-10T20:30:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:10,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:30:10,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/2f996b112f7c42b8b7ad3eacdffdea10 is 1080, key is row0097/info:/1731270610428/Put/seqid=0 2024-11-10T20:30:10,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741856_1032 (size=12516) 2024-11-10T20:30:10,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741856_1032 (size=12516) 2024-11-10T20:30:10,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/2f996b112f7c42b8b7ad3eacdffdea10 2024-11-10T20:30:10,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/2f996b112f7c42b8b7ad3eacdffdea10 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/2f996b112f7c42b8b7ad3eacdffdea10 2024-11-10T20:30:10,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/2f996b112f7c42b8b7ad3eacdffdea10, entries=7, sequenceid=137, filesize=12.2 K 2024-11-10T20:30:10,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=16.81 KB/17216 for 08af9de3a5999d7a5aa81e049047fc83 in 25ms, sequenceid=137, compaction requested=false 2024-11-10T20:30:10,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:10,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB 2024-11-10T20:30:10,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/7c9ebbb3329c476faa67361308163c8a is 1080, key is row0104/info:/1731270610439/Put/seqid=0 2024-11-10T20:30:10,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741857_1033 (size=23316) 2024-11-10T20:30:10,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741857_1033 (size=23316) 2024-11-10T20:30:10,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/7c9ebbb3329c476faa67361308163c8a 2024-11-10T20:30:10,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/7c9ebbb3329c476faa67361308163c8a as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/7c9ebbb3329c476faa67361308163c8a 2024-11-10T20:30:10,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/7c9ebbb3329c476faa67361308163c8a, entries=17, sequenceid=157, filesize=22.8 K 2024-11-10T20:30:10,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=8.41 KB/8608 for 08af9de3a5999d7a5aa81e049047fc83 in 22ms, sequenceid=157, compaction requested=true 2024-11-10T20:30:10,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:10,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08af9de3a5999d7a5aa81e049047fc83:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:30:10,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:10,486 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:30:10,487 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 78913 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:30:10,488 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1541): 08af9de3a5999d7a5aa81e049047fc83/info is initiating minor compaction (all files) 2024-11-10T20:30:10,488 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08af9de3a5999d7a5aa81e049047fc83/info in TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:10,488 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/657561f87134480f95db946a4e5dff8e, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/2f996b112f7c42b8b7ad3eacdffdea10, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/7c9ebbb3329c476faa67361308163c8a] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp, totalSize=77.1 K 2024-11-10T20:30:10,488 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 657561f87134480f95db946a4e5dff8e, keycount=35, bloomtype=ROW, size=42.1 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731270596307 2024-11-10T20:30:10,489 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f996b112f7c42b8b7ad3eacdffdea10, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731270610428 2024-11-10T20:30:10,489 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c9ebbb3329c476faa67361308163c8a, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731270610439 2024-11-10T20:30:10,500 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08af9de3a5999d7a5aa81e049047fc83#info#compaction#71 average throughput is 30.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:30:10,501 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/1ce28be3275e49eda899aeaf6e426b2f is 1080, key is row0062/info:/1731270596307/Put/seqid=0 2024-11-10T20:30:10,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741858_1034 (size=69123) 2024-11-10T20:30:10,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741858_1034 (size=69123) 2024-11-10T20:30:10,511 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/1ce28be3275e49eda899aeaf6e426b2f as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/1ce28be3275e49eda899aeaf6e426b2f 2024-11-10T20:30:10,518 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08af9de3a5999d7a5aa81e049047fc83/info of 08af9de3a5999d7a5aa81e049047fc83 into 1ce28be3275e49eda899aeaf6e426b2f(size=67.5 K), total size for store is 67.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:30:10,519 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:10,519 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., storeName=08af9de3a5999d7a5aa81e049047fc83/info, priority=13, startTime=1731270610486; duration=0sec 2024-11-10T20:30:10,519 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:10,519 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08af9de3a5999d7a5aa81e049047fc83:info 2024-11-10T20:30:11,251 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:11,251 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:12,252 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:12,252 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:12,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB 2024-11-10T20:30:12,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/8713638cac6e4673aa8277484749a734 is 1080, key is row0121/info:/1731270610465/Put/seqid=0 2024-11-10T20:30:12,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741859_1035 (size=14672) 2024-11-10T20:30:12,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741859_1035 (size=14672) 2024-11-10T20:30:12,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/8713638cac6e4673aa8277484749a734 2024-11-10T20:30:12,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/8713638cac6e4673aa8277484749a734 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/8713638cac6e4673aa8277484749a734 2024-11-10T20:30:12,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/8713638cac6e4673aa8277484749a734, entries=9, sequenceid=170, filesize=14.3 K 2024-11-10T20:30:12,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=14.71 KB/15064 for 08af9de3a5999d7a5aa81e049047fc83 in 22ms, sequenceid=170, compaction requested=false 2024-11-10T20:30:12,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:12,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T20:30:12,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/78272e2e482e4917b671c7ed5478d633 is 1080, key is row0130/info:/1731270612482/Put/seqid=0 2024-11-10T20:30:12,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741860_1036 (size=21156) 2024-11-10T20:30:12,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741860_1036 (size=21156) 2024-11-10T20:30:12,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/78272e2e482e4917b671c7ed5478d633 2024-11-10T20:30:12,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/78272e2e482e4917b671c7ed5478d633 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/78272e2e482e4917b671c7ed5478d633 2024-11-10T20:30:12,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/78272e2e482e4917b671c7ed5478d633, entries=15, sequenceid=188, filesize=20.7 K 2024-11-10T20:30:12,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=13.66 KB/13988 for 08af9de3a5999d7a5aa81e049047fc83 in 21ms, sequenceid=188, compaction requested=true 2024-11-10T20:30:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08af9de3a5999d7a5aa81e049047fc83:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:30:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:12,525 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:30:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:12,527 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 104951 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:30:12,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T20:30:12,527 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1541): 08af9de3a5999d7a5aa81e049047fc83/info is initiating minor compaction (all files) 2024-11-10T20:30:12,527 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08af9de3a5999d7a5aa81e049047fc83/info in TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:12,527 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/1ce28be3275e49eda899aeaf6e426b2f, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/8713638cac6e4673aa8277484749a734, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/78272e2e482e4917b671c7ed5478d633] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp, totalSize=102.5 K 2024-11-10T20:30:12,527 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce28be3275e49eda899aeaf6e426b2f, keycount=59, bloomtype=ROW, size=67.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731270596307 2024-11-10T20:30:12,528 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8713638cac6e4673aa8277484749a734, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731270610465 2024-11-10T20:30:12,528 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78272e2e482e4917b671c7ed5478d633, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731270612482 2024-11-10T20:30:12,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/5a447191e8d046e59a5fbce2ea16eb79 is 1080, key is row0145/info:/1731270612505/Put/seqid=0 2024-11-10T20:30:12,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741861_1037 (size=20078) 2024-11-10T20:30:12,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741861_1037 (size=20078) 2024-11-10T20:30:12,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/5a447191e8d046e59a5fbce2ea16eb79 2024-11-10T20:30:12,541 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08af9de3a5999d7a5aa81e049047fc83#info#compaction#75 average throughput is 42.59 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:30:12,542 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/f42a32ccb709431e8d8e28ff24d09ec4 is 1080, key is row0062/info:/1731270596307/Put/seqid=0 2024-11-10T20:30:12,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/5a447191e8d046e59a5fbce2ea16eb79 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5a447191e8d046e59a5fbce2ea16eb79 2024-11-10T20:30:12,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741862_1038 (size=95174) 2024-11-10T20:30:12,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741862_1038 (size=95174) 2024-11-10T20:30:12,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5a447191e8d046e59a5fbce2ea16eb79, entries=14, sequenceid=205, filesize=19.6 K 2024-11-10T20:30:12,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=2.10 KB/2152 for 08af9de3a5999d7a5aa81e049047fc83 in 22ms, sequenceid=205, compaction requested=false 2024-11-10T20:30:12,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:12,551 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/f42a32ccb709431e8d8e28ff24d09ec4 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/f42a32ccb709431e8d8e28ff24d09ec4 2024-11-10T20:30:12,556 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08af9de3a5999d7a5aa81e049047fc83/info of 08af9de3a5999d7a5aa81e049047fc83 into f42a32ccb709431e8d8e28ff24d09ec4(size=92.9 K), total size for store is 112.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:30:12,556 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:12,556 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., storeName=08af9de3a5999d7a5aa81e049047fc83/info, priority=13, startTime=1731270612525; duration=0sec 2024-11-10T20:30:12,556 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:12,556 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08af9de3a5999d7a5aa81e049047fc83:info 2024-11-10T20:30:13,253 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:13,253 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:14,253 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:14,253 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:14,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:30:14,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/3d9b909bcc2a4219ab26bb88f4b4fc8e is 1080, key is row0159/info:/1731270612528/Put/seqid=0 2024-11-10T20:30:14,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741863_1039 (size=12516) 2024-11-10T20:30:14,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741863_1039 (size=12516) 2024-11-10T20:30:14,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/3d9b909bcc2a4219ab26bb88f4b4fc8e 2024-11-10T20:30:14,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/3d9b909bcc2a4219ab26bb88f4b4fc8e as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/3d9b909bcc2a4219ab26bb88f4b4fc8e 2024-11-10T20:30:14,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/3d9b909bcc2a4219ab26bb88f4b4fc8e, entries=7, sequenceid=216, filesize=12.2 K 2024-11-10T20:30:14,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 08af9de3a5999d7a5aa81e049047fc83 in 21ms, sequenceid=216, compaction requested=true 2024-11-10T20:30:14,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:14,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08af9de3a5999d7a5aa81e049047fc83:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:30:14,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:14,560 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:30:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:14,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T20:30:14,561 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127768 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:30:14,561 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1541): 08af9de3a5999d7a5aa81e049047fc83/info is initiating minor compaction (all files) 2024-11-10T20:30:14,561 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08af9de3a5999d7a5aa81e049047fc83/info in TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:14,561 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/f42a32ccb709431e8d8e28ff24d09ec4, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5a447191e8d046e59a5fbce2ea16eb79, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/3d9b909bcc2a4219ab26bb88f4b4fc8e] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp, totalSize=124.8 K 2024-11-10T20:30:14,562 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting f42a32ccb709431e8d8e28ff24d09ec4, keycount=83, bloomtype=ROW, size=92.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731270596307 2024-11-10T20:30:14,562 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a447191e8d046e59a5fbce2ea16eb79, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731270612505 2024-11-10T20:30:14,563 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d9b909bcc2a4219ab26bb88f4b4fc8e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731270612528 2024-11-10T20:30:14,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/5d58f774ecd04b689b3b75db36d78b4a is 1080, key is row0166/info:/1731270614539/Put/seqid=0 2024-11-10T20:30:14,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741864_1040 (size=21156) 2024-11-10T20:30:14,575 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08af9de3a5999d7a5aa81e049047fc83#info#compaction#78 average throughput is 53.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:30:14,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741864_1040 (size=21156) 2024-11-10T20:30:14,576 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/0ba183d793a749e79c8210c2ba088908 is 1080, key is row0062/info:/1731270596307/Put/seqid=0 2024-11-10T20:30:14,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741865_1041 (size=117918) 2024-11-10T20:30:14,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741865_1041 (size=117918) 2024-11-10T20:30:14,586 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/0ba183d793a749e79c8210c2ba088908 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/0ba183d793a749e79c8210c2ba088908 2024-11-10T20:30:14,591 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08af9de3a5999d7a5aa81e049047fc83/info of 08af9de3a5999d7a5aa81e049047fc83 into 0ba183d793a749e79c8210c2ba088908(size=115.2 K), total size for store is 115.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:30:14,591 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:14,591 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., storeName=08af9de3a5999d7a5aa81e049047fc83/info, priority=13, startTime=1731270614560; duration=0sec 2024-11-10T20:30:14,591 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:14,591 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08af9de3a5999d7a5aa81e049047fc83:info 2024-11-10T20:30:14,889 INFO [master/734d44df782b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T20:30:14,890 INFO [master/734d44df782b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T20:30:14,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/5d58f774ecd04b689b3b75db36d78b4a 2024-11-10T20:30:14,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/5d58f774ecd04b689b3b75db36d78b4a as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5d58f774ecd04b689b3b75db36d78b4a 2024-11-10T20:30:14,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5d58f774ecd04b689b3b75db36d78b4a, entries=15, sequenceid=234, filesize=20.7 K 2024-11-10T20:30:14,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=12.61 KB/12912 for 08af9de3a5999d7a5aa81e049047fc83 in 427ms, sequenceid=234, compaction requested=false 2024-11-10T20:30:14,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:15,254 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:15,254 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:16,254 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:16,254 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:16,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-11-10T20:30:16,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/c6889d99dd4c4d6c9c68c920755fb352 is 1080, key is row0181/info:/1731270614561/Put/seqid=0 2024-11-10T20:30:16,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741866_1042 (size=19000) 2024-11-10T20:30:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741866_1042 (size=19000) 2024-11-10T20:30:16,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/c6889d99dd4c4d6c9c68c920755fb352 2024-11-10T20:30:16,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/c6889d99dd4c4d6c9c68c920755fb352 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/c6889d99dd4c4d6c9c68c920755fb352 2024-11-10T20:30:16,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/c6889d99dd4c4d6c9c68c920755fb352, entries=13, sequenceid=251, filesize=18.6 K 2024-11-10T20:30:16,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=14.71 KB/15064 for 08af9de3a5999d7a5aa81e049047fc83 in 24ms, sequenceid=251, compaction requested=true 2024-11-10T20:30:16,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:16,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08af9de3a5999d7a5aa81e049047fc83:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:30:16,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:16,608 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:30:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:16,609 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 158074 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:30:16,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T20:30:16,609 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1541): 08af9de3a5999d7a5aa81e049047fc83/info is initiating minor compaction (all files) 2024-11-10T20:30:16,609 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08af9de3a5999d7a5aa81e049047fc83/info in TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:16,609 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/0ba183d793a749e79c8210c2ba088908, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5d58f774ecd04b689b3b75db36d78b4a, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/c6889d99dd4c4d6c9c68c920755fb352] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp, totalSize=154.4 K 2024-11-10T20:30:16,610 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ba183d793a749e79c8210c2ba088908, keycount=104, bloomtype=ROW, size=115.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731270596307 2024-11-10T20:30:16,610 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d58f774ecd04b689b3b75db36d78b4a, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731270614539 2024-11-10T20:30:16,611 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6889d99dd4c4d6c9c68c920755fb352, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731270614561 2024-11-10T20:30:16,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/4f856993fadc491a8375b4c3a96f3019 is 1080, key is row0194/info:/1731270616584/Put/seqid=0 2024-11-10T20:30:16,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741867_1043 (size=21168) 2024-11-10T20:30:16,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741867_1043 (size=21168) 2024-11-10T20:30:16,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/4f856993fadc491a8375b4c3a96f3019 2024-11-10T20:30:16,626 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08af9de3a5999d7a5aa81e049047fc83#info#compaction#81 average throughput is 45.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:30:16,626 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/5f85724be9d949a682754332856a5e16 is 1080, key is row0062/info:/1731270596307/Put/seqid=0 2024-11-10T20:30:16,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/4f856993fadc491a8375b4c3a96f3019 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/4f856993fadc491a8375b4c3a96f3019 2024-11-10T20:30:16,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/4f856993fadc491a8375b4c3a96f3019, entries=15, sequenceid=269, filesize=20.7 K 2024-11-10T20:30:16,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=13.66 KB/13988 for 08af9de3a5999d7a5aa81e049047fc83 in 26ms, sequenceid=269, compaction requested=false 2024-11-10T20:30:16,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:16,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741868_1044 (size=148409) 2024-11-10T20:30:16,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741868_1044 (size=148409) 2024-11-10T20:30:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:16,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T20:30:16,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/819779372260481988c9ff24044e0e8d is 1080, key is row0209/info:/1731270616610/Put/seqid=0 2024-11-10T20:30:16,642 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/5f85724be9d949a682754332856a5e16 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5f85724be9d949a682754332856a5e16 2024-11-10T20:30:16,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741869_1045 (size=20092) 2024-11-10T20:30:16,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741869_1045 (size=20092) 2024-11-10T20:30:16,649 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08af9de3a5999d7a5aa81e049047fc83/info of 08af9de3a5999d7a5aa81e049047fc83 into 5f85724be9d949a682754332856a5e16(size=144.9 K), total size for store is 165.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:30:16,649 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:16,649 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., storeName=08af9de3a5999d7a5aa81e049047fc83/info, priority=13, startTime=1731270616607; duration=0sec 2024-11-10T20:30:16,649 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:16,649 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08af9de3a5999d7a5aa81e049047fc83:info 2024-11-10T20:30:17,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/819779372260481988c9ff24044e0e8d 2024-11-10T20:30:17,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/819779372260481988c9ff24044e0e8d as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/819779372260481988c9ff24044e0e8d 2024-11-10T20:30:17,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/819779372260481988c9ff24044e0e8d, entries=14, sequenceid=286, filesize=19.6 K 2024-11-10T20:30:17,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=2.10 KB/2152 for 08af9de3a5999d7a5aa81e049047fc83 in 425ms, sequenceid=286, compaction requested=true 2024-11-10T20:30:17,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:17,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08af9de3a5999d7a5aa81e049047fc83:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:30:17,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:17,061 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:30:17,062 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 189669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:30:17,062 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1541): 08af9de3a5999d7a5aa81e049047fc83/info is initiating minor compaction (all files) 2024-11-10T20:30:17,062 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08af9de3a5999d7a5aa81e049047fc83/info in TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:17,062 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5f85724be9d949a682754332856a5e16, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/4f856993fadc491a8375b4c3a96f3019, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/819779372260481988c9ff24044e0e8d] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp, totalSize=185.2 K 2024-11-10T20:30:17,063 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f85724be9d949a682754332856a5e16, keycount=132, bloomtype=ROW, size=144.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731270596307 2024-11-10T20:30:17,063 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f856993fadc491a8375b4c3a96f3019, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731270616584 2024-11-10T20:30:17,063 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 819779372260481988c9ff24044e0e8d, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731270616610 2024-11-10T20:30:17,074 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08af9de3a5999d7a5aa81e049047fc83#info#compaction#83 average throughput is 55.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:30:17,075 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/e96d5f5ae2544377a088eb4610608fbb is 1080, key is row0062/info:/1731270596307/Put/seqid=0 2024-11-10T20:30:17,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741870_1046 (size=179807) 2024-11-10T20:30:17,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741870_1046 (size=179807) 2024-11-10T20:30:17,083 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/e96d5f5ae2544377a088eb4610608fbb as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/e96d5f5ae2544377a088eb4610608fbb 2024-11-10T20:30:17,088 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08af9de3a5999d7a5aa81e049047fc83/info of 08af9de3a5999d7a5aa81e049047fc83 into e96d5f5ae2544377a088eb4610608fbb(size=175.6 K), total size for store is 175.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:30:17,088 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:17,088 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., storeName=08af9de3a5999d7a5aa81e049047fc83/info, priority=13, startTime=1731270617061; duration=0sec 2024-11-10T20:30:17,088 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:17,088 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08af9de3a5999d7a5aa81e049047fc83:info 2024-11-10T20:30:17,255 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:17,255 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:18,255 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:18,255 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:18,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T20:30:18,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/a046e3aba7cb4dfab3febe834a59f4b4 is 1080, key is row0223/info:/1731270616637/Put/seqid=0 2024-11-10T20:30:18,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741871_1047 (size=12523) 2024-11-10T20:30:18,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741871_1047 (size=12523) 2024-11-10T20:30:18,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/a046e3aba7cb4dfab3febe834a59f4b4 2024-11-10T20:30:18,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/a046e3aba7cb4dfab3febe834a59f4b4 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/a046e3aba7cb4dfab3febe834a59f4b4 2024-11-10T20:30:18,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/a046e3aba7cb4dfab3febe834a59f4b4, entries=7, sequenceid=298, filesize=12.2 K 2024-11-10T20:30:18,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 08af9de3a5999d7a5aa81e049047fc83 in 22ms, sequenceid=298, compaction requested=false 2024-11-10T20:30:18,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44413 {}] regionserver.HRegion(8855): Flush requested on 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:18,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T20:30:18,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/28e627a2db9e4c58b1fab4aa303c8410 is 1080, key is row0230/info:/1731270618649/Put/seqid=0 2024-11-10T20:30:18,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741872_1048 (size=21171) 2024-11-10T20:30:18,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741872_1048 (size=21171) 2024-11-10T20:30:18,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/28e627a2db9e4c58b1fab4aa303c8410 2024-11-10T20:30:18,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/28e627a2db9e4c58b1fab4aa303c8410 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/28e627a2db9e4c58b1fab4aa303c8410 2024-11-10T20:30:18,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/28e627a2db9e4c58b1fab4aa303c8410, entries=15, sequenceid=316, filesize=20.7 K 2024-11-10T20:30:18,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=12.61 KB/12912 for 08af9de3a5999d7a5aa81e049047fc83 in 22ms, sequenceid=316, compaction requested=true 2024-11-10T20:30:18,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:18,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 08af9de3a5999d7a5aa81e049047fc83:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T20:30:18,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:18,693 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T20:30:18,694 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 213501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T20:30:18,694 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1541): 08af9de3a5999d7a5aa81e049047fc83/info is initiating minor compaction (all files) 2024-11-10T20:30:18,694 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 08af9de3a5999d7a5aa81e049047fc83/info in TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:18,694 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/e96d5f5ae2544377a088eb4610608fbb, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/a046e3aba7cb4dfab3febe834a59f4b4, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/28e627a2db9e4c58b1fab4aa303c8410] into tmpdir=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp, totalSize=208.5 K 2024-11-10T20:30:18,695 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e96d5f5ae2544377a088eb4610608fbb, keycount=161, bloomtype=ROW, size=175.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731270596307 2024-11-10T20:30:18,695 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a046e3aba7cb4dfab3febe834a59f4b4, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731270616637 2024-11-10T20:30:18,695 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28e627a2db9e4c58b1fab4aa303c8410, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731270618649 2024-11-10T20:30:18,707 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 08af9de3a5999d7a5aa81e049047fc83#info#compaction#86 average throughput is 62.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T20:30:18,707 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/05b3e9e8e64a47e180064ec998e7640e is 1080, key is row0062/info:/1731270596307/Put/seqid=0 2024-11-10T20:30:18,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741873_1049 (size=203655) 2024-11-10T20:30:18,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741873_1049 (size=203655) 2024-11-10T20:30:18,717 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/05b3e9e8e64a47e180064ec998e7640e as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/05b3e9e8e64a47e180064ec998e7640e 2024-11-10T20:30:18,723 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 08af9de3a5999d7a5aa81e049047fc83/info of 08af9de3a5999d7a5aa81e049047fc83 into 05b3e9e8e64a47e180064ec998e7640e(size=198.9 K), total size for store is 198.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T20:30:18,723 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:18,723 INFO [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., storeName=08af9de3a5999d7a5aa81e049047fc83/info, priority=13, startTime=1731270618693; duration=0sec 2024-11-10T20:30:18,723 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T20:30:18,723 DEBUG [RS:0;734d44df782b:44413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 08af9de3a5999d7a5aa81e049047fc83:info 2024-11-10T20:30:18,967 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375 2024-11-10T20:30:19,256 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:19,256 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:20,257 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:20,257 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:20,690 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-11-10T20:30:20,690 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C44413%2C1731270573213.1731270620690 2024-11-10T20:30:20,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,696 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.1731270573610 with entries=310, filesize=307.89 KB; new WAL /user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.1731270620690 2024-11-10T20:30:20,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42933:42933),(127.0.0.1/127.0.0.1:35135:35135)] 2024-11-10T20:30:20,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.1731270573610 is not closed yet, will try archiving it next time 2024-11-10T20:30:20,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741833_1009 (size=315283) 2024-11-10T20:30:20,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741833_1009 (size=315283) 2024-11-10T20:30:20,702 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 08af9de3a5999d7a5aa81e049047fc83 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-10T20:30:20,706 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/77b7b543e0a8465688e779c983e05b88 is 1080, key is row0245/info:/1731270618672/Put/seqid=0 2024-11-10T20:30:20,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741875_1051 (size=17918) 2024-11-10T20:30:20,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741875_1051 (size=17918) 2024-11-10T20:30:20,711 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/77b7b543e0a8465688e779c983e05b88 2024-11-10T20:30:20,716 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/.tmp/info/77b7b543e0a8465688e779c983e05b88 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/77b7b543e0a8465688e779c983e05b88 2024-11-10T20:30:20,721 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/77b7b543e0a8465688e779c983e05b88, entries=12, sequenceid=332, filesize=17.5 K 2024-11-10T20:30:20,722 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for 08af9de3a5999d7a5aa81e049047fc83 in 21ms, sequenceid=332, compaction requested=false 2024-11-10T20:30:20,722 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 08af9de3a5999d7a5aa81e049047fc83: 2024-11-10T20:30:20,722 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dc40cbb5ebd30c95e94857ec7ee91b25: 2024-11-10T20:30:20,722 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB 2024-11-10T20:30:20,726 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/info/45c04f8e5fbf455e8f3bfbd30b7ab299 is 186, key is TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25./info:regioninfo/1731270599089/Put/seqid=0 2024-11-10T20:30:20,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741876_1052 (size=6153) 2024-11-10T20:30:20,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741876_1052 (size=6153) 2024-11-10T20:30:20,733 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/info/45c04f8e5fbf455e8f3bfbd30b7ab299 2024-11-10T20:30:20,738 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/.tmp/info/45c04f8e5fbf455e8f3bfbd30b7ab299 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/info/45c04f8e5fbf455e8f3bfbd30b7ab299 2024-11-10T20:30:20,742 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/info/45c04f8e5fbf455e8f3bfbd30b7ab299, entries=5, sequenceid=21, filesize=6.0 K 2024-11-10T20:30:20,743 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 21ms, sequenceid=21, compaction requested=false 2024-11-10T20:30:20,743 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-10T20:30:20,743 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C44413%2C1731270573213.1731270620743 2024-11-10T20:30:20,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:20,749 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.1731270620690 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.1731270620743 2024-11-10T20:30:20,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42933:42933),(127.0.0.1/127.0.0.1:35135:35135)] 2024-11-10T20:30:20,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.1731270620690 is not closed yet, will try archiving it next time 2024-11-10T20:30:20,750 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.1731270573610 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/oldWALs/734d44df782b%2C44413%2C1731270573213.1731270573610 2024-11-10T20:30:20,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741874_1050 (size=731) 2024-11-10T20:30:20,751 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T20:30:20,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741874_1050 (size=731) 2024-11-10T20:30:20,752 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/WALs/734d44df782b,44413,1731270573213/734d44df782b%2C44413%2C1731270573213.1731270620690 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/oldWALs/734d44df782b%2C44413%2C1731270573213.1731270620690 2024-11-10T20:30:20,851 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T20:30:20,851 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:30:20,851 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:30:20,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:20,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:20,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T20:30:20,852 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T20:30:20,852 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1898102628, stopped=false 2024-11-10T20:30:20,852 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=734d44df782b,35643,1731270573164 2024-11-10T20:30:20,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:30:20,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:30:20,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:20,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:20,855 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:30:20,856 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:30:20,856 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:30:20,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:20,856 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '734d44df782b,44413,1731270573213' ***** 2024-11-10T20:30:20,856 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T20:30:20,856 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:30:20,856 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:30:20,856 INFO [RS:0;734d44df782b:44413 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T20:30:20,857 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(3091): Received CLOSE for 08af9de3a5999d7a5aa81e049047fc83 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(3091): Received CLOSE for dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(959): stopping server 734d44df782b,44413,1731270573213 2024-11-10T20:30:20,857 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 08af9de3a5999d7a5aa81e049047fc83, disabling compactions & flushes 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:30:20,857 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:20,857 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;734d44df782b:44413. 2024-11-10T20:30:20,857 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. after waiting 0 ms 2024-11-10T20:30:20,857 DEBUG [RS:0;734d44df782b:44413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:30:20,857 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:20,857 DEBUG [RS:0;734d44df782b:44413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T20:30:20,857 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-10T20:30:20,858 DEBUG [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(1325): Online Regions={08af9de3a5999d7a5aa81e049047fc83=TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83., dc40cbb5ebd30c95e94857ec7ee91b25=TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T20:30:20,858 DEBUG [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(1351): Waiting on 08af9de3a5999d7a5aa81e049047fc83, 1588230740, dc40cbb5ebd30c95e94857ec7ee91b25 2024-11-10T20:30:20,858 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:30:20,858 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:30:20,858 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:30:20,858 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:30:20,858 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:30:20,858 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248->hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/46e22e59335046bebad71837e99ec257-top, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-f9889582fdb94b3cbe19f422990db53e, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-acfd7de9e7e34a6ba6161b50cc3710eb, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/657561f87134480f95db946a4e5dff8e, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-ad9617199a414f8c95d21c7777eb772c, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/2f996b112f7c42b8b7ad3eacdffdea10, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/1ce28be3275e49eda899aeaf6e426b2f, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/7c9ebbb3329c476faa67361308163c8a, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/8713638cac6e4673aa8277484749a734, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/f42a32ccb709431e8d8e28ff24d09ec4, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/78272e2e482e4917b671c7ed5478d633, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5a447191e8d046e59a5fbce2ea16eb79, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/0ba183d793a749e79c8210c2ba088908, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/3d9b909bcc2a4219ab26bb88f4b4fc8e, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5d58f774ecd04b689b3b75db36d78b4a, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5f85724be9d949a682754332856a5e16, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/c6889d99dd4c4d6c9c68c920755fb352, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/4f856993fadc491a8375b4c3a96f3019, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/e96d5f5ae2544377a088eb4610608fbb, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/819779372260481988c9ff24044e0e8d, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/a046e3aba7cb4dfab3febe834a59f4b4, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/28e627a2db9e4c58b1fab4aa303c8410] to archive 2024-11-10T20:30:20,859 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T20:30:20,861 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248 2024-11-10T20:30:20,862 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-f9889582fdb94b3cbe19f422990db53e to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-f9889582fdb94b3cbe19f422990db53e 2024-11-10T20:30:20,863 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-10T20:30:20,863 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:30:20,863 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:30:20,863 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270620858Running coprocessor pre-close hooks at 1731270620858Disabling compacts and flushes for region at 1731270620858Disabling writes for close at 1731270620858Writing region close event to WAL at 1731270620859 (+1 ms)Running coprocessor post-close hooks at 1731270620863 (+4 ms)Closed at 1731270620863 2024-11-10T20:30:20,864 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T20:30:20,864 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-acfd7de9e7e34a6ba6161b50cc3710eb to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-acfd7de9e7e34a6ba6161b50cc3710eb 2024-11-10T20:30:20,865 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/657561f87134480f95db946a4e5dff8e to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/657561f87134480f95db946a4e5dff8e 2024-11-10T20:30:20,866 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-ad9617199a414f8c95d21c7777eb772c to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/TestLogRolling-testLogRolling=8be33adc55ab4770839f9c195768a248-ad9617199a414f8c95d21c7777eb772c 2024-11-10T20:30:20,867 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/2f996b112f7c42b8b7ad3eacdffdea10 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/2f996b112f7c42b8b7ad3eacdffdea10 2024-11-10T20:30:20,868 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/1ce28be3275e49eda899aeaf6e426b2f to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/1ce28be3275e49eda899aeaf6e426b2f 2024-11-10T20:30:20,869 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/7c9ebbb3329c476faa67361308163c8a to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/7c9ebbb3329c476faa67361308163c8a 2024-11-10T20:30:20,870 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/8713638cac6e4673aa8277484749a734 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/8713638cac6e4673aa8277484749a734 2024-11-10T20:30:20,871 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/f42a32ccb709431e8d8e28ff24d09ec4 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/f42a32ccb709431e8d8e28ff24d09ec4 2024-11-10T20:30:20,872 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/78272e2e482e4917b671c7ed5478d633 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/78272e2e482e4917b671c7ed5478d633 2024-11-10T20:30:20,874 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5a447191e8d046e59a5fbce2ea16eb79 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5a447191e8d046e59a5fbce2ea16eb79 2024-11-10T20:30:20,875 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/0ba183d793a749e79c8210c2ba088908 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/0ba183d793a749e79c8210c2ba088908 2024-11-10T20:30:20,876 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/3d9b909bcc2a4219ab26bb88f4b4fc8e to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/3d9b909bcc2a4219ab26bb88f4b4fc8e 2024-11-10T20:30:20,877 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5d58f774ecd04b689b3b75db36d78b4a to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5d58f774ecd04b689b3b75db36d78b4a 2024-11-10T20:30:20,878 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5f85724be9d949a682754332856a5e16 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/5f85724be9d949a682754332856a5e16 2024-11-10T20:30:20,879 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/c6889d99dd4c4d6c9c68c920755fb352 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/c6889d99dd4c4d6c9c68c920755fb352 2024-11-10T20:30:20,880 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/4f856993fadc491a8375b4c3a96f3019 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/4f856993fadc491a8375b4c3a96f3019 2024-11-10T20:30:20,881 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/e96d5f5ae2544377a088eb4610608fbb to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/e96d5f5ae2544377a088eb4610608fbb 2024-11-10T20:30:20,882 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/819779372260481988c9ff24044e0e8d to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/819779372260481988c9ff24044e0e8d 2024-11-10T20:30:20,883 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/a046e3aba7cb4dfab3febe834a59f4b4 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/a046e3aba7cb4dfab3febe834a59f4b4 2024-11-10T20:30:20,884 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/28e627a2db9e4c58b1fab4aa303c8410 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/info/28e627a2db9e4c58b1fab4aa303c8410 2024-11-10T20:30:20,885 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=734d44df782b:35643 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-10T20:30:20,885 WARN [StoreCloser-TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [657561f87134480f95db946a4e5dff8e=43081, 2f996b112f7c42b8b7ad3eacdffdea10=12516, 1ce28be3275e49eda899aeaf6e426b2f=69123, 7c9ebbb3329c476faa67361308163c8a=23316, 8713638cac6e4673aa8277484749a734=14672, f42a32ccb709431e8d8e28ff24d09ec4=95174, 78272e2e482e4917b671c7ed5478d633=21156, 5a447191e8d046e59a5fbce2ea16eb79=20078, 0ba183d793a749e79c8210c2ba088908=117918, 3d9b909bcc2a4219ab26bb88f4b4fc8e=12516, 5d58f774ecd04b689b3b75db36d78b4a=21156, 5f85724be9d949a682754332856a5e16=148409, c6889d99dd4c4d6c9c68c920755fb352=19000, 4f856993fadc491a8375b4c3a96f3019=21168, e96d5f5ae2544377a088eb4610608fbb=179807, 819779372260481988c9ff24044e0e8d=20092, a046e3aba7cb4dfab3febe834a59f4b4=12523, 28e627a2db9e4c58b1fab4aa303c8410=21171] 2024-11-10T20:30:20,889 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/08af9de3a5999d7a5aa81e049047fc83/recovered.edits/335.seqid, newMaxSeqId=335, maxSeqId=126 2024-11-10T20:30:20,889 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:20,889 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 08af9de3a5999d7a5aa81e049047fc83: Waiting for close lock at 1731270620857Running coprocessor pre-close hooks at 1731270620857Disabling compacts and flushes for region at 1731270620857Disabling writes for close at 1731270620857Writing region close event to WAL at 1731270620885 (+28 ms)Running coprocessor post-close hooks at 1731270620889 (+4 ms)Closed at 1731270620889 2024-11-10T20:30:20,890 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731270598385.08af9de3a5999d7a5aa81e049047fc83. 2024-11-10T20:30:20,890 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dc40cbb5ebd30c95e94857ec7ee91b25, disabling compactions & flushes 2024-11-10T20:30:20,890 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:30:20,890 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:30:20,890 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. after waiting 0 ms 2024-11-10T20:30:20,890 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:30:20,890 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248->hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/8be33adc55ab4770839f9c195768a248/info/46e22e59335046bebad71837e99ec257-bottom] to archive 2024-11-10T20:30:20,891 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T20:30:20,892 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248 to hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/archive/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/info/46e22e59335046bebad71837e99ec257.8be33adc55ab4770839f9c195768a248 2024-11-10T20:30:20,892 WARN [StoreCloser-TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-11-10T20:30:20,896 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/data/default/TestLogRolling-testLogRolling/dc40cbb5ebd30c95e94857ec7ee91b25/recovered.edits/131.seqid, newMaxSeqId=131, maxSeqId=126 2024-11-10T20:30:20,896 INFO [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:30:20,896 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dc40cbb5ebd30c95e94857ec7ee91b25: Waiting for close lock at 1731270620890Running coprocessor pre-close hooks at 1731270620890Disabling compacts and flushes for region at 1731270620890Disabling writes for close at 1731270620890Writing region close event to WAL at 1731270620893 (+3 ms)Running coprocessor post-close hooks at 1731270620896 (+3 ms)Closed at 1731270620896 2024-11-10T20:30:20,897 DEBUG [RS_CLOSE_REGION-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731270598385.dc40cbb5ebd30c95e94857ec7ee91b25. 2024-11-10T20:30:21,058 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(976): stopping server 734d44df782b,44413,1731270573213; all regions closed. 2024-11-10T20:30:21,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741834_1010 (size=8107) 2024-11-10T20:30:21,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741834_1010 (size=8107) 2024-11-10T20:30:21,063 DEBUG [RS:0;734d44df782b:44413 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/oldWALs 2024-11-10T20:30:21,063 INFO [RS:0;734d44df782b:44413 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C44413%2C1731270573213.meta:.meta(num 1731270573959) 2024-11-10T20:30:21,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741877_1053 (size=780) 2024-11-10T20:30:21,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741877_1053 (size=780) 2024-11-10T20:30:21,067 DEBUG [RS:0;734d44df782b:44413 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/oldWALs 2024-11-10T20:30:21,067 INFO [RS:0;734d44df782b:44413 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C44413%2C1731270573213:(num 1731270620743) 2024-11-10T20:30:21,067 DEBUG [RS:0;734d44df782b:44413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:21,067 INFO [RS:0;734d44df782b:44413 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:30:21,067 INFO [RS:0;734d44df782b:44413 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:30:21,067 INFO [RS:0;734d44df782b:44413 {}] hbase.ChoreService(370): Chore service for: regionserver/734d44df782b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T20:30:21,068 INFO [RS:0;734d44df782b:44413 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:30:21,068 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:30:21,068 INFO [RS:0;734d44df782b:44413 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44413 2024-11-10T20:30:21,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:30:21,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/734d44df782b,44413,1731270573213 2024-11-10T20:30:21,070 INFO [RS:0;734d44df782b:44413 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:30:21,071 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [734d44df782b,44413,1731270573213] 2024-11-10T20:30:21,072 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/734d44df782b,44413,1731270573213 already deleted, retry=false 2024-11-10T20:30:21,072 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 734d44df782b,44413,1731270573213 expired; onlineServers=0 2024-11-10T20:30:21,072 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '734d44df782b,35643,1731270573164' ***** 2024-11-10T20:30:21,072 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T20:30:21,072 INFO [M:0;734d44df782b:35643 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:30:21,072 INFO [M:0;734d44df782b:35643 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:30:21,072 DEBUG [M:0;734d44df782b:35643 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T20:30:21,072 DEBUG [M:0;734d44df782b:35643 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T20:30:21,072 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T20:30:21,072 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270573356 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270573356,5,FailOnTimeoutGroup] 2024-11-10T20:30:21,072 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270573360 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270573360,5,FailOnTimeoutGroup] 2024-11-10T20:30:21,072 INFO [M:0;734d44df782b:35643 {}] hbase.ChoreService(370): Chore service for: master/734d44df782b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T20:30:21,072 INFO [M:0;734d44df782b:35643 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:30:21,072 DEBUG [M:0;734d44df782b:35643 {}] master.HMaster(1795): Stopping service threads 2024-11-10T20:30:21,073 INFO [M:0;734d44df782b:35643 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T20:30:21,073 INFO [M:0;734d44df782b:35643 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:30:21,073 ERROR [M:0;734d44df782b:35643 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36749,5,PEWorkerGroup] 2024-11-10T20:30:21,073 INFO [M:0;734d44df782b:35643 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T20:30:21,073 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T20:30:21,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T20:30:21,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:21,074 DEBUG [M:0;734d44df782b:35643 {}] zookeeper.ZKUtil(347): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T20:30:21,074 WARN [M:0;734d44df782b:35643 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T20:30:21,074 INFO [M:0;734d44df782b:35643 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/.lastflushedseqids 2024-11-10T20:30:21,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741878_1054 (size=228) 2024-11-10T20:30:21,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741878_1054 (size=228) 2024-11-10T20:30:21,079 INFO [M:0;734d44df782b:35643 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T20:30:21,080 INFO [M:0;734d44df782b:35643 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T20:30:21,080 DEBUG [M:0;734d44df782b:35643 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:30:21,080 INFO [M:0;734d44df782b:35643 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:21,080 DEBUG [M:0;734d44df782b:35643 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:21,080 DEBUG [M:0;734d44df782b:35643 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:30:21,080 DEBUG [M:0;734d44df782b:35643 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:21,080 INFO [M:0;734d44df782b:35643 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=53.71 KB heapSize=65.93 KB 2024-11-10T20:30:21,095 DEBUG [M:0;734d44df782b:35643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c812b27d8fa49c7ae46996f5812037c is 82, key is hbase:meta,,1/info:regioninfo/1731270573983/Put/seqid=0 2024-11-10T20:30:21,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741879_1055 (size=5672) 2024-11-10T20:30:21,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741879_1055 (size=5672) 2024-11-10T20:30:21,099 INFO [M:0;734d44df782b:35643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c812b27d8fa49c7ae46996f5812037c 2024-11-10T20:30:21,118 DEBUG [M:0;734d44df782b:35643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e1e2029ad641497ea349985617d05ab2 is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731270574405/Put/seqid=0 2024-11-10T20:30:21,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741880_1056 (size=7681) 2024-11-10T20:30:21,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741880_1056 (size=7681) 2024-11-10T20:30:21,124 INFO [M:0;734d44df782b:35643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.11 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e1e2029ad641497ea349985617d05ab2 2024-11-10T20:30:21,128 INFO [M:0;734d44df782b:35643 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e1e2029ad641497ea349985617d05ab2 2024-11-10T20:30:21,141 DEBUG [M:0;734d44df782b:35643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40bcf20a300448feb02e7ec540f3df21 is 69, key is 734d44df782b,44413,1731270573213/rs:state/1731270573446/Put/seqid=0 2024-11-10T20:30:21,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741881_1057 (size=5156) 2024-11-10T20:30:21,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741881_1057 (size=5156) 2024-11-10T20:30:21,147 INFO [M:0;734d44df782b:35643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40bcf20a300448feb02e7ec540f3df21 2024-11-10T20:30:21,165 DEBUG [M:0;734d44df782b:35643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/611cde15374246e7b313e260d6963a0c is 52, key is load_balancer_on/state:d/1731270574036/Put/seqid=0 2024-11-10T20:30:21,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741882_1058 (size=5056) 2024-11-10T20:30:21,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741882_1058 (size=5056) 2024-11-10T20:30:21,170 INFO [M:0;734d44df782b:35643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/611cde15374246e7b313e260d6963a0c 2024-11-10T20:30:21,171 INFO [RS:0;734d44df782b:44413 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:30:21,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:30:21,171 INFO [RS:0;734d44df782b:44413 {}] regionserver.HRegionServer(1031): Exiting; stopping=734d44df782b,44413,1731270573213; zookeeper connection closed. 2024-11-10T20:30:21,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44413-0x10193dff1c70001, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:30:21,171 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21c54d87 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21c54d87 2024-11-10T20:30:21,172 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T20:30:21,175 DEBUG [M:0;734d44df782b:35643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c812b27d8fa49c7ae46996f5812037c as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8c812b27d8fa49c7ae46996f5812037c 2024-11-10T20:30:21,179 INFO [M:0;734d44df782b:35643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8c812b27d8fa49c7ae46996f5812037c, entries=8, sequenceid=129, filesize=5.5 K 2024-11-10T20:30:21,179 DEBUG [M:0;734d44df782b:35643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e1e2029ad641497ea349985617d05ab2 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e1e2029ad641497ea349985617d05ab2 2024-11-10T20:30:21,183 INFO [M:0;734d44df782b:35643 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e1e2029ad641497ea349985617d05ab2 2024-11-10T20:30:21,183 INFO [M:0;734d44df782b:35643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e1e2029ad641497ea349985617d05ab2, entries=14, sequenceid=129, filesize=7.5 K 2024-11-10T20:30:21,184 DEBUG [M:0;734d44df782b:35643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40bcf20a300448feb02e7ec540f3df21 as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/40bcf20a300448feb02e7ec540f3df21 2024-11-10T20:30:21,188 INFO [M:0;734d44df782b:35643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/40bcf20a300448feb02e7ec540f3df21, entries=1, sequenceid=129, filesize=5.0 K 2024-11-10T20:30:21,189 DEBUG [M:0;734d44df782b:35643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/611cde15374246e7b313e260d6963a0c as hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/611cde15374246e7b313e260d6963a0c 2024-11-10T20:30:21,193 INFO [M:0;734d44df782b:35643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36749/user/jenkins/test-data/eb2fd10b-f096-8981-f581-030a8a69c868/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/611cde15374246e7b313e260d6963a0c, entries=1, sequenceid=129, filesize=4.9 K 2024-11-10T20:30:21,194 INFO [M:0;734d44df782b:35643 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.71 KB/54997, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 114ms, sequenceid=129, compaction requested=false 2024-11-10T20:30:21,196 INFO [M:0;734d44df782b:35643 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:21,196 DEBUG [M:0;734d44df782b:35643 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270621080Disabling compacts and flushes for region at 1731270621080Disabling writes for close at 1731270621080Obtaining lock to block concurrent updates at 1731270621080Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731270621080Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=54997, getHeapSize=67448, getOffHeapSize=0, getCellsCount=152 at 1731270621080Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731270621081 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731270621081Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731270621094 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731270621094Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731270621103 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731270621118 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731270621118Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731270621128 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731270621141 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731270621141Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731270621151 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731270621164 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731270621164Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7714c9d1: reopening flushed file at 1731270621174 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3685368e: reopening flushed file at 1731270621179 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d2f7569: reopening flushed file at 1731270621184 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58412fbe: reopening flushed file at 1731270621188 (+4 ms)Finished flush of dataSize ~53.71 KB/54997, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 114ms, sequenceid=129, compaction requested=false at 1731270621194 (+6 ms)Writing region close event to WAL at 1731270621196 (+2 ms)Closed at 1731270621196 2024-11-10T20:30:21,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:21,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45981 is added to blk_1073741830_1006 (size=63927) 2024-11-10T20:30:21,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34313 is added to blk_1073741830_1006 (size=63927) 2024-11-10T20:30:21,199 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:30:21,199 INFO [M:0;734d44df782b:35643 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T20:30:21,199 INFO [M:0;734d44df782b:35643 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35643 2024-11-10T20:30:21,200 INFO [M:0;734d44df782b:35643 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:30:21,257 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:21,257 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:21,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:30:21,301 INFO [M:0;734d44df782b:35643 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:30:21,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35643-0x10193dff1c70000, quorum=127.0.0.1:58834, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:30:21,304 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@20d96a0d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:30:21,304 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4ac7d52f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:30:21,304 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:30:21,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4881a2ed{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:30:21,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45bda0cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.log.dir/,STOPPED} 2024-11-10T20:30:21,306 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:30:21,306 WARN [BP-1481773930-172.17.0.2-1731270572511 heartbeating to localhost/127.0.0.1:36749 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:30:21,306 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:30:21,306 WARN [BP-1481773930-172.17.0.2-1731270572511 heartbeating to localhost/127.0.0.1:36749 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1481773930-172.17.0.2-1731270572511 (Datanode Uuid ba5f09c2-703e-45d2-9d0e-db807413166b) service to localhost/127.0.0.1:36749 2024-11-10T20:30:21,307 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/data/data3/current/BP-1481773930-172.17.0.2-1731270572511 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:30:21,307 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/data/data4/current/BP-1481773930-172.17.0.2-1731270572511 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:30:21,307 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:30:21,309 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10e59f6e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:30:21,310 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@256ffd79{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:30:21,310 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:30:21,310 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@362b84a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:30:21,310 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@319ddf98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.log.dir/,STOPPED} 2024-11-10T20:30:21,311 WARN [BP-1481773930-172.17.0.2-1731270572511 heartbeating to localhost/127.0.0.1:36749 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:30:21,311 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:30:21,311 WARN [BP-1481773930-172.17.0.2-1731270572511 heartbeating to localhost/127.0.0.1:36749 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1481773930-172.17.0.2-1731270572511 (Datanode Uuid 6f6016a4-7b89-4587-80db-4865e4c81ea7) service to localhost/127.0.0.1:36749 2024-11-10T20:30:21,311 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:30:21,312 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/data/data1/current/BP-1481773930-172.17.0.2-1731270572511 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:30:21,312 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/cluster_270be98c-9121-a197-fdbd-7c56c2328984/data/data2/current/BP-1481773930-172.17.0.2-1731270572511 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:30:21,313 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:30:21,319 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@11e6950{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:30:21,319 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@53fc2fb7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:30:21,319 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:30:21,319 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7113b5b4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:30:21,320 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70e7b313{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.log.dir/,STOPPED} 2024-11-10T20:30:21,326 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T20:30:21,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T20:30:21,363 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=231 (was 206) Potentially hanging thread: nioEventLoopGroup-41-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36749 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36749 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36749 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36749 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:36749 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:36749 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36749 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36749 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36749 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-38-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=512 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=23 (was 28), ProcessCount=11 (was 11), AvailableMemoryMB=7182 (was 7265) 2024-11-10T20:30:21,373 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=231, OpenFileDescriptor=512, MaxFileDescriptor=1048576, SystemLoadAverage=23, ProcessCount=11, AvailableMemoryMB=7182 2024-11-10T20:30:21,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.log.dir so I do NOT create it in target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/86721941-e86f-274c-b4d3-27be669a4775/hadoop.tmp.dir so I do NOT create it in target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde, deleteOnExit=true 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/test.cache.data in system properties and HBase conf 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/hadoop.log.dir in system properties and HBase conf 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T20:30:21,374 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:30:21,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/nfs.dump.dir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/java.io.tmpdir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T20:30:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T20:30:21,388 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:30:21,442 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:30:21,445 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:30:21,446 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:30:21,446 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:30:21,446 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:30:21,447 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:30:21,448 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bb3701b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:30:21,448 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2fdb4cd1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:30:21,480 INFO [regionserver/734d44df782b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:30:21,562 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43f8fe5e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/java.io.tmpdir/jetty-localhost-39581-hadoop-hdfs-3_4_1-tests_jar-_-any-9167259987507111486/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:30:21,562 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c797b08{HTTP/1.1, (http/1.1)}{localhost:39581} 2024-11-10T20:30:21,562 INFO [Time-limited test {}] server.Server(415): Started @284037ms 2024-11-10T20:30:21,575 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T20:30:21,621 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:30:21,624 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:30:21,625 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:30:21,625 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:30:21,625 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T20:30:21,625 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dcbbc83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:30:21,626 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57de154f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:30:21,741 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e290644{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/java.io.tmpdir/jetty-localhost-42749-hadoop-hdfs-3_4_1-tests_jar-_-any-10174428856616367486/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:30:21,741 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e709428{HTTP/1.1, (http/1.1)}{localhost:42749} 2024-11-10T20:30:21,742 INFO [Time-limited test {}] server.Server(415): Started @284216ms 2024-11-10T20:30:21,743 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:30:21,771 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T20:30:21,773 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T20:30:21,774 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T20:30:21,774 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T20:30:21,774 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T20:30:21,774 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69a2ae1b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/hadoop.log.dir/,AVAILABLE} 2024-11-10T20:30:21,774 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2482618b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T20:30:21,849 WARN [Thread-2461 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/data/data1/current/BP-1922965003-172.17.0.2-1731270621395/current, will proceed with Du for space computation calculation, 2024-11-10T20:30:21,849 WARN [Thread-2462 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/data/data2/current/BP-1922965003-172.17.0.2-1731270621395/current, will proceed with Du for space computation calculation, 2024-11-10T20:30:21,867 WARN [Thread-2440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:30:21,869 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f272b9fe69be611 with lease ID 0x537064f509d3bcab: Processing first storage report for DS-7f2621d7-ef4f-4460-b009-09cac62619d3 from datanode DatanodeRegistration(127.0.0.1:40767, datanodeUuid=389515b7-e340-4f3c-b9c0-6584a040877c, infoPort=46803, infoSecurePort=0, ipcPort=46747, storageInfo=lv=-57;cid=testClusterID;nsid=1461959028;c=1731270621395) 2024-11-10T20:30:21,869 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f272b9fe69be611 with lease ID 0x537064f509d3bcab: from storage DS-7f2621d7-ef4f-4460-b009-09cac62619d3 node DatanodeRegistration(127.0.0.1:40767, datanodeUuid=389515b7-e340-4f3c-b9c0-6584a040877c, infoPort=46803, infoSecurePort=0, ipcPort=46747, storageInfo=lv=-57;cid=testClusterID;nsid=1461959028;c=1731270621395), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:30:21,869 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9f272b9fe69be611 with lease ID 0x537064f509d3bcab: Processing first storage report for DS-3253f1ed-4be9-4fea-a5a6-a74f14d8970b from datanode DatanodeRegistration(127.0.0.1:40767, datanodeUuid=389515b7-e340-4f3c-b9c0-6584a040877c, infoPort=46803, infoSecurePort=0, ipcPort=46747, storageInfo=lv=-57;cid=testClusterID;nsid=1461959028;c=1731270621395) 2024-11-10T20:30:21,869 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9f272b9fe69be611 with lease ID 0x537064f509d3bcab: from storage DS-3253f1ed-4be9-4fea-a5a6-a74f14d8970b node DatanodeRegistration(127.0.0.1:40767, datanodeUuid=389515b7-e340-4f3c-b9c0-6584a040877c, infoPort=46803, infoSecurePort=0, ipcPort=46747, storageInfo=lv=-57;cid=testClusterID;nsid=1461959028;c=1731270621395), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:30:21,907 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d664f93{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/java.io.tmpdir/jetty-localhost-45727-hadoop-hdfs-3_4_1-tests_jar-_-any-1505703043578390516/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:30:21,907 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@96dff09{HTTP/1.1, (http/1.1)}{localhost:45727} 2024-11-10T20:30:21,907 INFO [Time-limited test {}] server.Server(415): Started @284381ms 2024-11-10T20:30:21,908 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T20:30:21,999 WARN [Thread-2487 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/data/data3/current/BP-1922965003-172.17.0.2-1731270621395/current, will proceed with Du for space computation calculation, 2024-11-10T20:30:22,000 WARN [Thread-2488 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/data/data4/current/BP-1922965003-172.17.0.2-1731270621395/current, will proceed with Du for space computation calculation, 2024-11-10T20:30:22,016 WARN [Thread-2476 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T20:30:22,018 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1219dfb05b555d01 with lease ID 0x537064f509d3bcac: Processing first storage report for DS-767f338e-ba74-4917-91a1-5e1e66888165 from datanode DatanodeRegistration(127.0.0.1:36253, datanodeUuid=473c01cd-1c6b-4659-94e8-a63fab9c9f6c, infoPort=46597, infoSecurePort=0, ipcPort=41713, storageInfo=lv=-57;cid=testClusterID;nsid=1461959028;c=1731270621395) 2024-11-10T20:30:22,018 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1219dfb05b555d01 with lease ID 0x537064f509d3bcac: from storage DS-767f338e-ba74-4917-91a1-5e1e66888165 node DatanodeRegistration(127.0.0.1:36253, datanodeUuid=473c01cd-1c6b-4659-94e8-a63fab9c9f6c, infoPort=46597, infoSecurePort=0, ipcPort=41713, storageInfo=lv=-57;cid=testClusterID;nsid=1461959028;c=1731270621395), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:30:22,018 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1219dfb05b555d01 with lease ID 0x537064f509d3bcac: Processing first storage report for DS-eb7f1105-cb3a-4856-8bd3-d184dacc5724 from datanode DatanodeRegistration(127.0.0.1:36253, datanodeUuid=473c01cd-1c6b-4659-94e8-a63fab9c9f6c, infoPort=46597, infoSecurePort=0, ipcPort=41713, storageInfo=lv=-57;cid=testClusterID;nsid=1461959028;c=1731270621395) 2024-11-10T20:30:22,018 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1219dfb05b555d01 with lease ID 0x537064f509d3bcac: from storage DS-eb7f1105-cb3a-4856-8bd3-d184dacc5724 node DatanodeRegistration(127.0.0.1:36253, datanodeUuid=473c01cd-1c6b-4659-94e8-a63fab9c9f6c, infoPort=46597, infoSecurePort=0, ipcPort=41713, storageInfo=lv=-57;cid=testClusterID;nsid=1461959028;c=1731270621395), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T20:30:22,029 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e 2024-11-10T20:30:22,031 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/zookeeper_0, clientPort=54475, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T20:30:22,032 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54475 2024-11-10T20:30:22,032 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:30:22,034 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:30:22,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:30:22,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741825_1001 (size=7) 2024-11-10T20:30:22,042 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41 with version=8 2024-11-10T20:30:22,042 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41563/user/jenkins/test-data/d0c2c58a-ffaf-99a7-9011-14e3dde8704b/hbase-staging 2024-11-10T20:30:22,044 INFO [Time-limited test {}] client.ConnectionUtils(128): master/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:30:22,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:30:22,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:30:22,044 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:30:22,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:30:22,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:30:22,044 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T20:30:22,045 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:30:22,045 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33205 2024-11-10T20:30:22,046 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33205 connecting to ZooKeeper ensemble=127.0.0.1:54475 2024-11-10T20:30:22,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:332050x0, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:30:22,052 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33205-0x10193e0b0b70000 connected 2024-11-10T20:30:22,071 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:30:22,072 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:30:22,074 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:30:22,074 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41, hbase.cluster.distributed=false 2024-11-10T20:30:22,076 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:30:22,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33205 2024-11-10T20:30:22,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33205 2024-11-10T20:30:22,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33205 2024-11-10T20:30:22,077 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33205 2024-11-10T20:30:22,077 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33205 2024-11-10T20:30:22,091 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/734d44df782b:0 server-side Connection retries=45 2024-11-10T20:30:22,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:30:22,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T20:30:22,092 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T20:30:22,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T20:30:22,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T20:30:22,092 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T20:30:22,092 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T20:30:22,093 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33437 2024-11-10T20:30:22,093 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33437 connecting to ZooKeeper ensemble=127.0.0.1:54475 2024-11-10T20:30:22,094 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:30:22,095 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:30:22,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334370x0, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T20:30:22,103 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:334370x0, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:30:22,103 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33437-0x10193e0b0b70001 connected 2024-11-10T20:30:22,103 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T20:30:22,104 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T20:30:22,104 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T20:30:22,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T20:30:22,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33437 2024-11-10T20:30:22,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33437 2024-11-10T20:30:22,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33437 2024-11-10T20:30:22,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33437 2024-11-10T20:30:22,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33437 2024-11-10T20:30:22,121 DEBUG [M:0;734d44df782b:33205 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;734d44df782b:33205 2024-11-10T20:30:22,122 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/734d44df782b,33205,1731270622044 2024-11-10T20:30:22,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:30:22,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:30:22,124 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/734d44df782b,33205,1731270622044 2024-11-10T20:30:22,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T20:30:22,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,126 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T20:30:22,126 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/734d44df782b,33205,1731270622044 from backup master directory 2024-11-10T20:30:22,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/734d44df782b,33205,1731270622044 2024-11-10T20:30:22,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:30:22,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T20:30:22,127 WARN [master/734d44df782b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:30:22,127 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=734d44df782b,33205,1731270622044 2024-11-10T20:30:22,133 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/hbase.id] with ID: 6011edd0-fc51-4b69-bce5-24125f83eeb8 2024-11-10T20:30:22,133 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/.tmp/hbase.id 2024-11-10T20:30:22,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:30:22,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741826_1002 (size=42) 2024-11-10T20:30:22,141 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/.tmp/hbase.id]:[hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/hbase.id] 2024-11-10T20:30:22,151 INFO [master/734d44df782b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:30:22,151 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T20:30:22,152 INFO [master/734d44df782b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T20:30:22,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:30:22,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741827_1003 (size=196) 2024-11-10T20:30:22,160 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T20:30:22,161 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T20:30:22,161 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:30:22,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:30:22,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741828_1004 (size=1189) 2024-11-10T20:30:22,170 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store 2024-11-10T20:30:22,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:30:22,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741829_1005 (size=34) 2024-11-10T20:30:22,176 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:30:22,176 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:30:22,176 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:22,176 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:22,176 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:30:22,176 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:22,176 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:22,176 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270622176Disabling compacts and flushes for region at 1731270622176Disabling writes for close at 1731270622176Writing region close event to WAL at 1731270622176Closed at 1731270622176 2024-11-10T20:30:22,177 WARN [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/.initializing 2024-11-10T20:30:22,177 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/WALs/734d44df782b,33205,1731270622044 2024-11-10T20:30:22,179 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C33205%2C1731270622044, suffix=, logDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/WALs/734d44df782b,33205,1731270622044, archiveDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/oldWALs, maxLogs=10 2024-11-10T20:30:22,180 INFO [master/734d44df782b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33205%2C1731270622044.1731270622179 2024-11-10T20:30:22,183 INFO [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/WALs/734d44df782b,33205,1731270622044/734d44df782b%2C33205%2C1731270622044.1731270622179 2024-11-10T20:30:22,189 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46803:46803),(127.0.0.1/127.0.0.1:46597:46597)] 2024-11-10T20:30:22,189 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:30:22,190 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:30:22,190 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,190 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T20:30:22,192 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:30:22,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T20:30:22,194 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:30:22,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,195 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T20:30:22,195 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,195 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:30:22,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T20:30:22,196 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,197 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T20:30:22,197 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,198 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,198 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,199 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,199 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,199 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T20:30:22,200 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T20:30:22,202 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:30:22,202 INFO [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=857477, jitterRate=0.09033912420272827}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T20:30:22,203 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731270622190Initializing all the Stores at 1731270622191 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270622191Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270622191Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270622191Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270622191Cleaning up temporary data from old regions at 1731270622199 (+8 ms)Region opened successfully at 1731270622203 (+4 ms) 2024-11-10T20:30:22,203 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T20:30:22,206 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ecbe3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:30:22,207 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T20:30:22,207 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T20:30:22,207 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T20:30:22,207 INFO [master/734d44df782b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T20:30:22,207 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T20:30:22,208 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T20:30:22,208 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T20:30:22,210 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T20:30:22,210 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T20:30:22,212 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T20:30:22,212 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T20:30:22,213 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T20:30:22,214 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T20:30:22,214 INFO [master/734d44df782b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T20:30:22,215 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T20:30:22,216 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T20:30:22,217 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T20:30:22,218 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T20:30:22,220 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T20:30:22,221 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T20:30:22,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:30:22,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T20:30:22,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,224 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=734d44df782b,33205,1731270622044, sessionid=0x10193e0b0b70000, setting cluster-up flag (Was=false) 2024-11-10T20:30:22,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,232 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T20:30:22,233 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,33205,1731270622044 2024-11-10T20:30:22,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,240 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T20:30:22,241 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=734d44df782b,33205,1731270622044 2024-11-10T20:30:22,242 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T20:30:22,243 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T20:30:22,243 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T20:30:22,244 INFO [master/734d44df782b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T20:30:22,244 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 734d44df782b,33205,1731270622044 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T20:30:22,245 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:30:22,245 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:30:22,245 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:30:22,245 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/734d44df782b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T20:30:22,245 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/734d44df782b:0, corePoolSize=10, maxPoolSize=10 2024-11-10T20:30:22,245 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,245 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:30:22,245 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,246 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:30:22,246 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731270652246 2024-11-10T20:30:22,246 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T20:30:22,247 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T20:30:22,247 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T20:30:22,247 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T20:30:22,247 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T20:30:22,247 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T20:30:22,247 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T20:30:22,247 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,247 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T20:30:22,247 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,247 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T20:30:22,248 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T20:30:22,247 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T20:30:22,248 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T20:30:22,248 INFO [master/734d44df782b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T20:30:22,248 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270622248,5,FailOnTimeoutGroup] 2024-11-10T20:30:22,249 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270622248,5,FailOnTimeoutGroup] 2024-11-10T20:30:22,249 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,249 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T20:30:22,249 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,249 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:30:22,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741831_1007 (size=1321) 2024-11-10T20:30:22,254 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T20:30:22,254 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41 2024-11-10T20:30:22,258 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:22,258 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:22,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:30:22,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741832_1008 (size=32) 2024-11-10T20:30:22,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:30:22,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:30:22,265 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:30:22,265 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:30:22,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:30:22,267 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:30:22,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:30:22,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:30:22,268 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:30:22,268 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,269 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:30:22,269 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:30:22,270 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:30:22,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:30:22,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:30:22,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740 2024-11-10T20:30:22,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740 2024-11-10T20:30:22,273 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:30:22,273 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:30:22,273 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:30:22,274 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:30:22,276 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T20:30:22,276 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=732501, jitterRate=-0.06857702136039734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:30:22,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731270622263Initializing all the Stores at 1731270622263Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270622263Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270622264 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270622264Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270622264Cleaning up temporary data from old regions at 1731270622273 (+9 ms)Region opened successfully at 1731270622277 (+4 ms) 2024-11-10T20:30:22,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:30:22,277 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:30:22,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:30:22,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:30:22,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:30:22,277 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:30:22,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270622277Disabling compacts and flushes for region at 1731270622277Disabling writes for close at 1731270622277Writing region close event to WAL at 1731270622277Closed at 1731270622277 2024-11-10T20:30:22,278 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:30:22,278 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T20:30:22,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T20:30:22,279 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:30:22,280 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T20:30:22,311 INFO [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(746): ClusterId : 6011edd0-fc51-4b69-bce5-24125f83eeb8 2024-11-10T20:30:22,311 DEBUG [RS:0;734d44df782b:33437 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T20:30:22,314 DEBUG [RS:0;734d44df782b:33437 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T20:30:22,314 DEBUG [RS:0;734d44df782b:33437 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T20:30:22,316 DEBUG [RS:0;734d44df782b:33437 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T20:30:22,316 DEBUG [RS:0;734d44df782b:33437 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbf8af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=734d44df782b/172.17.0.2:0 2024-11-10T20:30:22,328 DEBUG [RS:0;734d44df782b:33437 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;734d44df782b:33437 2024-11-10T20:30:22,328 INFO [RS:0;734d44df782b:33437 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T20:30:22,328 INFO [RS:0;734d44df782b:33437 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T20:30:22,328 DEBUG [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T20:30:22,328 INFO [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(2659): reportForDuty to master=734d44df782b,33205,1731270622044 with port=33437, startcode=1731270622091 2024-11-10T20:30:22,329 DEBUG [RS:0;734d44df782b:33437 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T20:30:22,331 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39101, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T20:30:22,331 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33205 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 734d44df782b,33437,1731270622091 2024-11-10T20:30:22,331 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33205 {}] master.ServerManager(517): Registering regionserver=734d44df782b,33437,1731270622091 2024-11-10T20:30:22,333 DEBUG [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41 2024-11-10T20:30:22,333 DEBUG [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45423 2024-11-10T20:30:22,333 DEBUG [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T20:30:22,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:30:22,335 DEBUG [RS:0;734d44df782b:33437 {}] zookeeper.ZKUtil(111): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/734d44df782b,33437,1731270622091 2024-11-10T20:30:22,335 WARN [RS:0;734d44df782b:33437 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T20:30:22,335 INFO [RS:0;734d44df782b:33437 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:30:22,335 DEBUG [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/734d44df782b,33437,1731270622091 2024-11-10T20:30:22,336 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [734d44df782b,33437,1731270622091] 2024-11-10T20:30:22,340 INFO [RS:0;734d44df782b:33437 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T20:30:22,341 INFO [RS:0;734d44df782b:33437 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T20:30:22,342 INFO [RS:0;734d44df782b:33437 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T20:30:22,342 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,342 INFO [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T20:30:22,342 INFO [RS:0;734d44df782b:33437 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T20:30:22,343 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/734d44df782b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/734d44df782b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:30:22,343 DEBUG [RS:0;734d44df782b:33437 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/734d44df782b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T20:30:22,343 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,343 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,343 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,343 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,343 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,343 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33437,1731270622091-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:30:22,357 INFO [RS:0;734d44df782b:33437 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T20:30:22,357 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33437,1731270622091-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,358 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,358 INFO [RS:0;734d44df782b:33437 {}] regionserver.Replication(171): 734d44df782b,33437,1731270622091 started 2024-11-10T20:30:22,371 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,371 INFO [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(1482): Serving as 734d44df782b,33437,1731270622091, RpcServer on 734d44df782b/172.17.0.2:33437, sessionid=0x10193e0b0b70001 2024-11-10T20:30:22,371 DEBUG [RS:0;734d44df782b:33437 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T20:30:22,371 DEBUG [RS:0;734d44df782b:33437 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 734d44df782b,33437,1731270622091 2024-11-10T20:30:22,371 DEBUG [RS:0;734d44df782b:33437 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,33437,1731270622091' 2024-11-10T20:30:22,371 DEBUG [RS:0;734d44df782b:33437 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T20:30:22,372 DEBUG [RS:0;734d44df782b:33437 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T20:30:22,372 DEBUG [RS:0;734d44df782b:33437 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T20:30:22,372 DEBUG [RS:0;734d44df782b:33437 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T20:30:22,372 DEBUG [RS:0;734d44df782b:33437 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 734d44df782b,33437,1731270622091 2024-11-10T20:30:22,372 DEBUG [RS:0;734d44df782b:33437 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '734d44df782b,33437,1731270622091' 2024-11-10T20:30:22,372 DEBUG [RS:0;734d44df782b:33437 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T20:30:22,372 DEBUG [RS:0;734d44df782b:33437 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T20:30:22,373 DEBUG [RS:0;734d44df782b:33437 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T20:30:22,373 INFO [RS:0;734d44df782b:33437 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T20:30:22,373 INFO [RS:0;734d44df782b:33437 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T20:30:22,431 WARN [734d44df782b:33205 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T20:30:22,474 INFO [RS:0;734d44df782b:33437 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C33437%2C1731270622091, suffix=, logDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/734d44df782b,33437,1731270622091, archiveDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/oldWALs, maxLogs=32 2024-11-10T20:30:22,475 INFO [RS:0;734d44df782b:33437 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33437%2C1731270622091.1731270622475 2024-11-10T20:30:22,480 INFO [RS:0;734d44df782b:33437 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/734d44df782b,33437,1731270622091/734d44df782b%2C33437%2C1731270622091.1731270622475 2024-11-10T20:30:22,481 DEBUG [RS:0;734d44df782b:33437 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46803:46803),(127.0.0.1/127.0.0.1:46597:46597)] 2024-11-10T20:30:22,681 DEBUG [734d44df782b:33205 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T20:30:22,681 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=734d44df782b,33437,1731270622091 2024-11-10T20:30:22,683 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,33437,1731270622091, state=OPENING 2024-11-10T20:30:22,684 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T20:30:22,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,686 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:30:22,686 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:30:22,686 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T20:30:22,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,33437,1731270622091}] 2024-11-10T20:30:22,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:30:22,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T20:30:22,805 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-10T20:30:22,839 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T20:30:22,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53937, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T20:30:22,845 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T20:30:22,845 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:30:22,846 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=734d44df782b%2C33437%2C1731270622091.meta, suffix=.meta, logDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/734d44df782b,33437,1731270622091, archiveDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/oldWALs, maxLogs=32 2024-11-10T20:30:22,847 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 734d44df782b%2C33437%2C1731270622091.meta.1731270622847.meta 2024-11-10T20:30:22,852 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/734d44df782b,33437,1731270622091/734d44df782b%2C33437%2C1731270622091.meta.1731270622847.meta 2024-11-10T20:30:22,856 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46803:46803),(127.0.0.1/127.0.0.1:46597:46597)] 2024-11-10T20:30:22,860 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T20:30:22,861 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T20:30:22,861 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T20:30:22,861 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T20:30:22,861 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T20:30:22,861 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T20:30:22,861 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T20:30:22,861 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T20:30:22,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T20:30:22,863 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T20:30:22,863 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:30:22,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T20:30:22,864 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T20:30:22,864 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:30:22,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T20:30:22,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T20:30:22,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:30:22,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T20:30:22,866 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T20:30:22,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T20:30:22,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T20:30:22,866 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T20:30:22,867 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740 2024-11-10T20:30:22,868 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740 2024-11-10T20:30:22,869 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T20:30:22,869 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T20:30:22,870 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T20:30:22,871 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T20:30:22,872 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=839313, jitterRate=0.067242830991745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T20:30:22,872 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T20:30:22,873 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731270622861Writing region info on filesystem at 1731270622861Initializing all the Stores at 1731270622862 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270622862Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270622862Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731270622862Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731270622862Cleaning up temporary data from old regions at 1731270622869 (+7 ms)Running coprocessor post-open hooks at 1731270622872 (+3 ms)Region opened successfully at 1731270622873 (+1 ms) 2024-11-10T20:30:22,874 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731270622839 2024-11-10T20:30:22,876 DEBUG [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T20:30:22,877 INFO [RS_OPEN_META-regionserver/734d44df782b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T20:30:22,877 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=734d44df782b,33437,1731270622091 2024-11-10T20:30:22,878 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 734d44df782b,33437,1731270622091, state=OPEN 2024-11-10T20:30:22,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:30:22,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T20:30:22,884 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=734d44df782b,33437,1731270622091 2024-11-10T20:30:22,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:30:22,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T20:30:22,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T20:30:22,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=734d44df782b,33437,1731270622091 in 198 msec 2024-11-10T20:30:22,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T20:30:22,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-11-10T20:30:22,889 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T20:30:22,889 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T20:30:22,891 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:30:22,891 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,33437,1731270622091, seqNum=-1] 2024-11-10T20:30:22,891 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:30:22,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:30:22,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 654 msec 2024-11-10T20:30:22,898 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731270622898, completionTime=-1 2024-11-10T20:30:22,898 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T20:30:22,898 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731270682900 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731270742900 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33205,1731270622044-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33205,1731270622044-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33205,1731270622044-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-734d44df782b:33205, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,900 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,902 DEBUG [master/734d44df782b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T20:30:22,904 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.776sec 2024-11-10T20:30:22,905 INFO [master/734d44df782b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T20:30:22,905 INFO [master/734d44df782b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T20:30:22,905 INFO [master/734d44df782b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T20:30:22,905 INFO [master/734d44df782b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T20:30:22,905 INFO [master/734d44df782b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T20:30:22,905 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33205,1731270622044-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T20:30:22,905 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33205,1731270622044-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T20:30:22,907 DEBUG [master/734d44df782b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T20:30:22,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T20:30:22,907 INFO [master/734d44df782b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=734d44df782b,33205,1731270622044-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T20:30:22,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55931fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:30:22,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 734d44df782b,33205,-1 for getting cluster id 2024-11-10T20:30:22,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T20:30:22,913 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6011edd0-fc51-4b69-bce5-24125f83eeb8' 2024-11-10T20:30:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T20:30:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6011edd0-fc51-4b69-bce5-24125f83eeb8" 2024-11-10T20:30:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc3ba86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:30:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [734d44df782b,33205,-1] 2024-11-10T20:30:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T20:30:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:22,915 INFO [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T20:30:22,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@242dc355, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T20:30:22,916 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T20:30:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=734d44df782b,33437,1731270622091, seqNum=-1] 2024-11-10T20:30:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T20:30:22,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T20:30:22,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=734d44df782b,33205,1731270622044 2024-11-10T20:30:22,921 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T20:30:22,923 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T20:30:22,923 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T20:30:22,925 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/test.com,8080,1, archiveDir=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/oldWALs, maxLogs=32 2024-11-10T20:30:22,926 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731270622926 2024-11-10T20:30:22,931 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/test.com,8080,1/test.com%2C8080%2C1.1731270622926 2024-11-10T20:30:22,932 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46597:46597),(127.0.0.1/127.0.0.1:46803:46803)] 2024-11-10T20:30:22,940 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731270622939 2024-11-10T20:30:22,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,945 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/test.com,8080,1/test.com%2C8080%2C1.1731270622926 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/test.com,8080,1/test.com%2C8080%2C1.1731270622939 2024-11-10T20:30:22,949 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46597:46597),(127.0.0.1/127.0.0.1:46803:46803)] 2024-11-10T20:30:22,949 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/test.com,8080,1/test.com%2C8080%2C1.1731270622926 is not closed yet, will try archiving it next time 2024-11-10T20:30:22,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741835_1011 (size=93) 2024-11-10T20:30:22,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741835_1011 (size=93) 2024-11-10T20:30:22,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:22,950 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/WALs/test.com,8080,1/test.com%2C8080%2C1.1731270622926 to hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/oldWALs/test.com%2C8080%2C1.1731270622926 2024-11-10T20:30:22,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741836_1012 (size=93) 2024-11-10T20:30:22,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741836_1012 (size=93) 2024-11-10T20:30:22,954 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/oldWALs 2024-11-10T20:30:22,954 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731270622939) 2024-11-10T20:30:22,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T20:30:22,954 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:30:22,955 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:30:22,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:22,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:22,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T20:30:22,955 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T20:30:22,955 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1220517315, stopped=false 2024-11-10T20:30:22,955 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=734d44df782b,33205,1731270622044 2024-11-10T20:30:22,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:30:22,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,957 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:30:22,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T20:30:22,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:22,958 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T20:30:22,958 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:30:22,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:22,958 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '734d44df782b,33437,1731270622091' ***** 2024-11-10T20:30:22,958 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T20:30:22,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:30:22,958 INFO [RS:0;734d44df782b:33437 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T20:30:22,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T20:30:22,959 INFO [RS:0;734d44df782b:33437 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T20:30:22,959 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T20:30:22,959 INFO [RS:0;734d44df782b:33437 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T20:30:22,959 INFO [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(959): stopping server 734d44df782b,33437,1731270622091 2024-11-10T20:30:22,959 INFO [RS:0;734d44df782b:33437 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:30:22,959 INFO [RS:0;734d44df782b:33437 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;734d44df782b:33437. 2024-11-10T20:30:22,959 DEBUG [RS:0;734d44df782b:33437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T20:30:22,959 DEBUG [RS:0;734d44df782b:33437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:22,959 INFO [RS:0;734d44df782b:33437 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T20:30:22,959 INFO [RS:0;734d44df782b:33437 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T20:30:22,959 INFO [RS:0;734d44df782b:33437 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T20:30:22,959 INFO [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T20:30:22,960 INFO [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T20:30:22,960 DEBUG [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T20:30:22,960 DEBUG [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T20:30:22,960 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T20:30:22,960 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T20:30:22,960 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T20:30:22,960 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T20:30:22,960 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T20:30:22,960 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-10T20:30:22,975 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740/.tmp/ns/9cd74e6ebcb4425681f8dd3cabc24632 is 43, key is default/ns:d/1731270622893/Put/seqid=0 2024-11-10T20:30:22,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741837_1013 (size=5153) 2024-11-10T20:30:22,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741837_1013 (size=5153) 2024-11-10T20:30:22,981 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740/.tmp/ns/9cd74e6ebcb4425681f8dd3cabc24632 2024-11-10T20:30:22,987 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740/.tmp/ns/9cd74e6ebcb4425681f8dd3cabc24632 as hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740/ns/9cd74e6ebcb4425681f8dd3cabc24632 2024-11-10T20:30:22,992 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740/ns/9cd74e6ebcb4425681f8dd3cabc24632, entries=2, sequenceid=6, filesize=5.0 K 2024-11-10T20:30:22,993 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=6, compaction requested=false 2024-11-10T20:30:22,993 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T20:30:22,997 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T20:30:22,997 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T20:30:22,997 INFO [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T20:30:22,997 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731270622960Running coprocessor pre-close hooks at 1731270622960Disabling compacts and flushes for region at 1731270622960Disabling writes for close at 1731270622960Obtaining lock to block concurrent updates at 1731270622960Preparing flush snapshotting stores in 1588230740 at 1731270622960Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731270622960Flushing stores of hbase:meta,,1.1588230740 at 1731270622961 (+1 ms)Flushing 1588230740/ns: creating writer at 1731270622961Flushing 1588230740/ns: appending metadata at 1731270622975 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731270622975Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a9d541f: reopening flushed file at 1731270622987 (+12 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=6, compaction requested=false at 1731270622993 (+6 ms)Writing region close event to WAL at 1731270622994 (+1 ms)Running coprocessor post-close hooks at 1731270622997 (+3 ms)Closed at 1731270622997 2024-11-10T20:30:22,997 DEBUG [RS_CLOSE_META-regionserver/734d44df782b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T20:30:23,160 INFO [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(976): stopping server 734d44df782b,33437,1731270622091; all regions closed. 2024-11-10T20:30:23,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741834_1010 (size=1152) 2024-11-10T20:30:23,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741834_1010 (size=1152) 2024-11-10T20:30:23,165 DEBUG [RS:0;734d44df782b:33437 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/oldWALs 2024-11-10T20:30:23,165 INFO [RS:0;734d44df782b:33437 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C33437%2C1731270622091.meta:.meta(num 1731270622847) 2024-11-10T20:30:23,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741833_1009 (size=93) 2024-11-10T20:30:23,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741833_1009 (size=93) 2024-11-10T20:30:23,169 DEBUG [RS:0;734d44df782b:33437 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/oldWALs 2024-11-10T20:30:23,169 INFO [RS:0;734d44df782b:33437 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 734d44df782b%2C33437%2C1731270622091:(num 1731270622475) 2024-11-10T20:30:23,169 DEBUG [RS:0;734d44df782b:33437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T20:30:23,169 INFO [RS:0;734d44df782b:33437 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T20:30:23,169 INFO [RS:0;734d44df782b:33437 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:30:23,169 INFO [RS:0;734d44df782b:33437 {}] hbase.ChoreService(370): Chore service for: regionserver/734d44df782b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T20:30:23,170 INFO [RS:0;734d44df782b:33437 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:30:23,170 INFO [regionserver/734d44df782b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:30:23,170 INFO [RS:0;734d44df782b:33437 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33437 2024-11-10T20:30:23,172 INFO [RS:0;734d44df782b:33437 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:30:23,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T20:30:23,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/734d44df782b,33437,1731270622091 2024-11-10T20:30:23,175 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [734d44df782b,33437,1731270622091] 2024-11-10T20:30:23,176 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/734d44df782b,33437,1731270622091 already deleted, retry=false 2024-11-10T20:30:23,176 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 734d44df782b,33437,1731270622091 expired; onlineServers=0 2024-11-10T20:30:23,176 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '734d44df782b,33205,1731270622044' ***** 2024-11-10T20:30:23,176 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T20:30:23,176 INFO [M:0;734d44df782b:33205 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T20:30:23,176 INFO [M:0;734d44df782b:33205 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T20:30:23,176 DEBUG [M:0;734d44df782b:33205 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T20:30:23,176 DEBUG [M:0;734d44df782b:33205 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T20:30:23,176 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T20:30:23,176 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270622248 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.large.0-1731270622248,5,FailOnTimeoutGroup] 2024-11-10T20:30:23,177 DEBUG [master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270622248 {}] cleaner.HFileCleaner(306): Exit Thread[master/734d44df782b:0:becomeActiveMaster-HFileCleaner.small.0-1731270622248,5,FailOnTimeoutGroup] 2024-11-10T20:30:23,177 INFO [M:0;734d44df782b:33205 {}] hbase.ChoreService(370): Chore service for: master/734d44df782b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T20:30:23,177 INFO [M:0;734d44df782b:33205 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T20:30:23,177 DEBUG [M:0;734d44df782b:33205 {}] master.HMaster(1795): Stopping service threads 2024-11-10T20:30:23,177 INFO [M:0;734d44df782b:33205 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T20:30:23,177 INFO [M:0;734d44df782b:33205 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T20:30:23,177 INFO [M:0;734d44df782b:33205 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T20:30:23,177 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T20:30:23,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T20:30:23,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T20:30:23,179 DEBUG [M:0;734d44df782b:33205 {}] zookeeper.ZKUtil(347): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T20:30:23,179 WARN [M:0;734d44df782b:33205 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T20:30:23,179 INFO [M:0;734d44df782b:33205 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/.lastflushedseqids 2024-11-10T20:30:23,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741838_1014 (size=108) 2024-11-10T20:30:23,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741838_1014 (size=108) 2024-11-10T20:30:23,184 INFO [M:0;734d44df782b:33205 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T20:30:23,185 INFO [M:0;734d44df782b:33205 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T20:30:23,185 DEBUG [M:0;734d44df782b:33205 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T20:30:23,185 INFO [M:0;734d44df782b:33205 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:23,185 DEBUG [M:0;734d44df782b:33205 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:23,185 DEBUG [M:0;734d44df782b:33205 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T20:30:23,185 DEBUG [M:0;734d44df782b:33205 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:23,185 INFO [M:0;734d44df782b:33205 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-10T20:30:23,200 DEBUG [M:0;734d44df782b:33205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9220bed8b08441a9b582a07bb0f05f0e is 82, key is hbase:meta,,1/info:regioninfo/1731270622877/Put/seqid=0 2024-11-10T20:30:23,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741839_1015 (size=5672) 2024-11-10T20:30:23,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741839_1015 (size=5672) 2024-11-10T20:30:23,205 INFO [M:0;734d44df782b:33205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9220bed8b08441a9b582a07bb0f05f0e 2024-11-10T20:30:23,222 DEBUG [M:0;734d44df782b:33205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa6dfff1fc4c4ce993c2e909ec30852f is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731270622897/Put/seqid=0 2024-11-10T20:30:23,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741840_1016 (size=5275) 2024-11-10T20:30:23,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741840_1016 (size=5275) 2024-11-10T20:30:23,227 INFO [M:0;734d44df782b:33205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa6dfff1fc4c4ce993c2e909ec30852f 2024-11-10T20:30:23,244 DEBUG [M:0;734d44df782b:33205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ff15cad1b387473fa8b6d78026718428 is 69, key is 734d44df782b,33437,1731270622091/rs:state/1731270622331/Put/seqid=0 2024-11-10T20:30:23,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741841_1017 (size=5156) 2024-11-10T20:30:23,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741841_1017 (size=5156) 2024-11-10T20:30:23,248 INFO [M:0;734d44df782b:33205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ff15cad1b387473fa8b6d78026718428 2024-11-10T20:30:23,258 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,43225,1731270442226/734d44df782b%2C43225%2C1731270442226.1731270442416 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:23,258 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34715/user/jenkins/test-data/40ac79b4-4438-6c4f-513b-c7ed0bb865ad/WALs/734d44df782b,45329,1731270441266/734d44df782b%2C45329%2C1731270441266.meta.1731270442046.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T20:30:23,266 DEBUG [M:0;734d44df782b:33205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9a4077dbc5f04438a0c7565d6e1ee11e is 52, key is load_balancer_on/state:d/1731270622922/Put/seqid=0 2024-11-10T20:30:23,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741842_1018 (size=5056) 2024-11-10T20:30:23,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741842_1018 (size=5056) 2024-11-10T20:30:23,270 INFO [M:0;734d44df782b:33205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9a4077dbc5f04438a0c7565d6e1ee11e 2024-11-10T20:30:23,275 DEBUG [M:0;734d44df782b:33205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9220bed8b08441a9b582a07bb0f05f0e as hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9220bed8b08441a9b582a07bb0f05f0e 2024-11-10T20:30:23,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:30:23,275 INFO [RS:0;734d44df782b:33437 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:30:23,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33437-0x10193e0b0b70001, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:30:23,275 INFO [RS:0;734d44df782b:33437 {}] regionserver.HRegionServer(1031): Exiting; stopping=734d44df782b,33437,1731270622091; zookeeper connection closed. 2024-11-10T20:30:23,275 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1d2f6a58 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1d2f6a58 2024-11-10T20:30:23,275 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T20:30:23,279 INFO [M:0;734d44df782b:33205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9220bed8b08441a9b582a07bb0f05f0e, entries=8, sequenceid=29, filesize=5.5 K 2024-11-10T20:30:23,280 DEBUG [M:0;734d44df782b:33205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa6dfff1fc4c4ce993c2e909ec30852f as hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa6dfff1fc4c4ce993c2e909ec30852f 2024-11-10T20:30:23,283 INFO [M:0;734d44df782b:33205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa6dfff1fc4c4ce993c2e909ec30852f, entries=3, sequenceid=29, filesize=5.2 K 2024-11-10T20:30:23,284 DEBUG [M:0;734d44df782b:33205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ff15cad1b387473fa8b6d78026718428 as hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ff15cad1b387473fa8b6d78026718428 2024-11-10T20:30:23,288 INFO [M:0;734d44df782b:33205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ff15cad1b387473fa8b6d78026718428, entries=1, sequenceid=29, filesize=5.0 K 2024-11-10T20:30:23,288 DEBUG [M:0;734d44df782b:33205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9a4077dbc5f04438a0c7565d6e1ee11e as hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9a4077dbc5f04438a0c7565d6e1ee11e 2024-11-10T20:30:23,292 INFO [M:0;734d44df782b:33205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45423/user/jenkins/test-data/a49b9091-c99b-de30-63f9-6d54c6257c41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9a4077dbc5f04438a0c7565d6e1ee11e, entries=1, sequenceid=29, filesize=4.9 K 2024-11-10T20:30:23,293 INFO [M:0;734d44df782b:33205 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=29, compaction requested=false 2024-11-10T20:30:23,294 INFO [M:0;734d44df782b:33205 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T20:30:23,294 DEBUG [M:0;734d44df782b:33205 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731270623185Disabling compacts and flushes for region at 1731270623185Disabling writes for close at 1731270623185Obtaining lock to block concurrent updates at 1731270623185Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731270623185Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731270623185Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731270623186 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731270623186Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731270623200 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731270623200Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731270623208 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731270623222 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731270623222Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731270623231 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731270623244 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731270623244Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731270623252 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731270623265 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731270623265Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14680101: reopening flushed file at 1731270623274 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74c2119c: reopening flushed file at 1731270623279 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ded9950: reopening flushed file at 1731270623283 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6113ab84: reopening flushed file at 1731270623288 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=29, compaction requested=false at 1731270623293 (+5 ms)Writing region close event to WAL at 1731270623294 (+1 ms)Closed at 1731270623294 2024-11-10T20:30:23,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T20:30:23,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36253 is added to blk_1073741830_1006 (size=10311) 2024-11-10T20:30:23,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40767 is added to blk_1073741830_1006 (size=10311) 2024-11-10T20:30:23,297 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T20:30:23,297 INFO [M:0;734d44df782b:33205 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T20:30:23,297 INFO [M:0;734d44df782b:33205 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33205 2024-11-10T20:30:23,298 INFO [M:0;734d44df782b:33205 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T20:30:23,400 INFO [M:0;734d44df782b:33205 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T20:30:23,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:30:23,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33205-0x10193e0b0b70000, quorum=127.0.0.1:54475, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T20:30:23,402 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d664f93{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:30:23,402 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@96dff09{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:30:23,403 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:30:23,403 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2482618b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:30:23,403 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69a2ae1b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/hadoop.log.dir/,STOPPED} 2024-11-10T20:30:23,404 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:30:23,404 WARN [BP-1922965003-172.17.0.2-1731270621395 heartbeating to localhost/127.0.0.1:45423 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:30:23,404 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:30:23,404 WARN [BP-1922965003-172.17.0.2-1731270621395 heartbeating to localhost/127.0.0.1:45423 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1922965003-172.17.0.2-1731270621395 (Datanode Uuid 473c01cd-1c6b-4659-94e8-a63fab9c9f6c) service to localhost/127.0.0.1:45423 2024-11-10T20:30:23,405 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/data/data3/current/BP-1922965003-172.17.0.2-1731270621395 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:30:23,405 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/data/data4/current/BP-1922965003-172.17.0.2-1731270621395 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:30:23,405 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:30:23,407 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e290644{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T20:30:23,407 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e709428{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:30:23,407 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:30:23,407 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57de154f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:30:23,407 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dcbbc83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/hadoop.log.dir/,STOPPED} 2024-11-10T20:30:23,408 WARN [BP-1922965003-172.17.0.2-1731270621395 heartbeating to localhost/127.0.0.1:45423 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T20:30:23,408 WARN [BP-1922965003-172.17.0.2-1731270621395 heartbeating to localhost/127.0.0.1:45423 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1922965003-172.17.0.2-1731270621395 (Datanode Uuid 389515b7-e340-4f3c-b9c0-6584a040877c) service to localhost/127.0.0.1:45423 2024-11-10T20:30:23,408 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T20:30:23,409 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T20:30:23,409 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/data/data1/current/BP-1922965003-172.17.0.2-1731270621395 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:30:23,409 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/cluster_e367ac7a-e026-84e1-309a-ece215982bde/data/data2/current/BP-1922965003-172.17.0.2-1731270621395 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T20:30:23,409 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T20:30:23,415 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43f8fe5e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T20:30:23,415 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c797b08{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T20:30:23,415 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T20:30:23,415 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2fdb4cd1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T20:30:23,415 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bb3701b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c3844fb9-f3f6-8f6b-5007-e5990172597e/hadoop.log.dir/,STOPPED} 2024-11-10T20:30:23,421 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T20:30:23,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T20:30:23,443 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=269 (was 231) Potentially hanging thread: nioEventLoopGroup-45-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: globalEventExecutor-1-19 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113) app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45423 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45423 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45423 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45423 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:45423 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45423 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45423 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins@localhost:45423 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=535 (was 512) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=23 (was 23), ProcessCount=11 (was 11), AvailableMemoryMB=7165 (was 7182)