2024-12-05 22:12:27,209 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc 2024-12-05 22:12:27,226 main DEBUG Took 0.014234 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 22:12:27,226 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 22:12:27,226 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 22:12:27,227 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 22:12:27,228 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,237 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 22:12:27,251 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,253 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,254 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,255 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,255 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,257 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,257 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,258 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,259 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,260 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,260 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,261 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,261 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,262 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,263 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,263 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,263 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,264 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 22:12:27,264 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,264 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 22:12:27,266 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 22:12:27,267 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 22:12:27,270 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 22:12:27,270 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 22:12:27,272 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 22:12:27,272 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 22:12:27,284 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 22:12:27,287 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 22:12:27,289 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 22:12:27,289 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 22:12:27,290 main DEBUG createAppenders(={Console}) 2024-12-05 22:12:27,291 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc initialized 2024-12-05 22:12:27,291 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc 2024-12-05 22:12:27,292 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc OK. 2024-12-05 22:12:27,292 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 22:12:27,293 main DEBUG OutputStream closed 2024-12-05 22:12:27,293 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 22:12:27,294 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 22:12:27,294 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@53ce1329 OK 2024-12-05 22:12:27,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 22:12:27,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 22:12:27,378 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 22:12:27,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 22:12:27,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 22:12:27,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 22:12:27,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 22:12:27,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 22:12:27,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 22:12:27,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 22:12:27,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 22:12:27,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 22:12:27,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 22:12:27,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 22:12:27,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 22:12:27,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 22:12:27,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 22:12:27,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 22:12:27,386 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 22:12:27,387 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-12-05 22:12:27,387 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 22:12:27,388 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-12-05T22:12:27,760 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc 2024-12-05 22:12:27,766 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 22:12:27,766 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T22:12:27,779 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-12-05T22:12:27,821 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=286, MaxFileDescriptor=1048576, SystemLoadAverage=637, ProcessCount=11, AvailableMemoryMB=5284 2024-12-05T22:12:27,824 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T22:12:27,827 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24, deleteOnExit=true 2024-12-05T22:12:27,827 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-05T22:12:27,828 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/test.cache.data in system properties and HBase conf 2024-12-05T22:12:27,828 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T22:12:27,829 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.log.dir in system properties and HBase conf 2024-12-05T22:12:27,829 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T22:12:27,830 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T22:12:27,830 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-05T22:12:27,931 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T22:12:28,059 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T22:12:28,064 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:12:28,065 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:12:28,065 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T22:12:28,066 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:12:28,066 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T22:12:28,067 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T22:12:28,068 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:12:28,068 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:12:28,069 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T22:12:28,069 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/nfs.dump.dir in system properties and HBase conf 2024-12-05T22:12:28,070 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/java.io.tmpdir in system properties and HBase conf 2024-12-05T22:12:28,070 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:12:28,071 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T22:12:28,071 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T22:12:28,626 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:12:29,055 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T22:12:29,143 INFO [Time-limited test {}] log.Log(170): Logging initialized @2859ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T22:12:29,220 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:12:29,311 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:12:29,353 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:12:29,354 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:12:29,356 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:12:29,373 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:12:29,377 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@88aab13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:12:29,378 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74468826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:12:29,635 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5682c4d1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/java.io.tmpdir/jetty-localhost-46177-hadoop-hdfs-3_4_1-tests_jar-_-any-6310012743133362226/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:12:29,660 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ff1a6c1{HTTP/1.1, (http/1.1)}{localhost:46177} 2024-12-05T22:12:29,660 INFO [Time-limited test {}] server.Server(415): Started @3378ms 2024-12-05T22:12:29,697 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:12:30,131 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:12:30,139 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:12:30,141 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:12:30,141 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:12:30,142 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:12:30,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2276bd44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:12:30,144 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b4ce9e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:12:30,314 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6aad8790{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/java.io.tmpdir/jetty-localhost-38487-hadoop-hdfs-3_4_1-tests_jar-_-any-8278283847827271599/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:12:30,315 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@587d1dca{HTTP/1.1, (http/1.1)}{localhost:38487} 2024-12-05T22:12:30,316 INFO [Time-limited test {}] server.Server(415): Started @4033ms 2024-12-05T22:12:30,381 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:12:30,535 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:12:30,543 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:12:30,546 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:12:30,547 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:12:30,547 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:12:30,551 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4debea22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:12:30,552 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6eb1b261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:12:30,712 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@163cfad6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/java.io.tmpdir/jetty-localhost-44195-hadoop-hdfs-3_4_1-tests_jar-_-any-12939710582794520031/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:12:30,713 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f952caa{HTTP/1.1, (http/1.1)}{localhost:44195} 2024-12-05T22:12:30,713 INFO [Time-limited test {}] server.Server(415): Started @4431ms 2024-12-05T22:12:30,716 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:12:31,016 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/dfs/data/data1/current/BP-716073671-172.17.0.2-1733436748740/current, will proceed with Du for space computation calculation, 2024-12-05T22:12:31,017 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/dfs/data/data2/current/BP-716073671-172.17.0.2-1733436748740/current, will proceed with Du for space computation calculation, 2024-12-05T22:12:31,020 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/dfs/data/data3/current/BP-716073671-172.17.0.2-1733436748740/current, will proceed with Du for space computation calculation, 2024-12-05T22:12:31,034 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/dfs/data/data4/current/BP-716073671-172.17.0.2-1733436748740/current, will proceed with Du for space computation calculation, 2024-12-05T22:12:31,104 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:12:31,108 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:12:31,211 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7867adad49626179 with lease ID 0xbb0fb2383833d215: Processing first storage report for DS-8cb37b21-2bb1-4287-8634-382232d8217e from datanode DatanodeRegistration(127.0.0.1:43775, datanodeUuid=6b1ac47c-29d5-4a6f-ac3b-1dd8e06757b3, infoPort=36315, infoSecurePort=0, ipcPort=34557, storageInfo=lv=-57;cid=testClusterID;nsid=686699879;c=1733436748740) 2024-12-05T22:12:31,213 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7867adad49626179 with lease ID 0xbb0fb2383833d215: from storage DS-8cb37b21-2bb1-4287-8634-382232d8217e node DatanodeRegistration(127.0.0.1:43775, datanodeUuid=6b1ac47c-29d5-4a6f-ac3b-1dd8e06757b3, infoPort=36315, infoSecurePort=0, ipcPort=34557, storageInfo=lv=-57;cid=testClusterID;nsid=686699879;c=1733436748740), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-05T22:12:31,213 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1dc1f3aaef2abaf4 with lease ID 0xbb0fb2383833d214: Processing first storage report for DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2 from datanode DatanodeRegistration(127.0.0.1:44383, datanodeUuid=aed8d442-5bd2-4df9-b6c0-f4129b88bed0, infoPort=36661, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-57;cid=testClusterID;nsid=686699879;c=1733436748740) 2024-12-05T22:12:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1dc1f3aaef2abaf4 with lease ID 0xbb0fb2383833d214: from storage DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2 node DatanodeRegistration(127.0.0.1:44383, datanodeUuid=aed8d442-5bd2-4df9-b6c0-f4129b88bed0, infoPort=36661, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-57;cid=testClusterID;nsid=686699879;c=1733436748740), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:12:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1dc1f3aaef2abaf4 with lease ID 0xbb0fb2383833d214: Processing first storage report for DS-9d00fbb1-2fb7-4856-8f82-5e8ff738156c from datanode DatanodeRegistration(127.0.0.1:44383, datanodeUuid=aed8d442-5bd2-4df9-b6c0-f4129b88bed0, infoPort=36661, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-57;cid=testClusterID;nsid=686699879;c=1733436748740) 2024-12-05T22:12:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1dc1f3aaef2abaf4 with lease ID 0xbb0fb2383833d214: from storage DS-9d00fbb1-2fb7-4856-8f82-5e8ff738156c node DatanodeRegistration(127.0.0.1:44383, datanodeUuid=aed8d442-5bd2-4df9-b6c0-f4129b88bed0, infoPort=36661, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-57;cid=testClusterID;nsid=686699879;c=1733436748740), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:12:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7867adad49626179 with lease ID 0xbb0fb2383833d215: Processing first storage report for DS-2ea0107e-89df-4a0d-9f92-d2239d8ef4cb from datanode DatanodeRegistration(127.0.0.1:43775, datanodeUuid=6b1ac47c-29d5-4a6f-ac3b-1dd8e06757b3, infoPort=36315, infoSecurePort=0, ipcPort=34557, storageInfo=lv=-57;cid=testClusterID;nsid=686699879;c=1733436748740) 2024-12-05T22:12:31,215 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7867adad49626179 with lease ID 0xbb0fb2383833d215: from storage DS-2ea0107e-89df-4a0d-9f92-d2239d8ef4cb node DatanodeRegistration(127.0.0.1:43775, datanodeUuid=6b1ac47c-29d5-4a6f-ac3b-1dd8e06757b3, infoPort=36315, infoSecurePort=0, ipcPort=34557, storageInfo=lv=-57;cid=testClusterID;nsid=686699879;c=1733436748740), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T22:12:31,293 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc 2024-12-05T22:12:31,401 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/zookeeper_0, clientPort=52878, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T22:12:31,414 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=52878 2024-12-05T22:12:31,441 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:12:31,445 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:12:31,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:12:31,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:12:31,801 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866 with version=8 2024-12-05T22:12:31,801 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/hbase-staging 2024-12-05T22:12:31,961 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T22:12:32,302 INFO [Time-limited test {}] client.ConnectionUtils(129): master/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:12:32,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:12:32,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:12:32,325 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:12:32,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:12:32,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:12:32,467 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:12:32,548 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T22:12:32,560 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T22:12:32,566 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:12:32,602 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 28783 (auto-detected) 2024-12-05T22:12:32,603 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T22:12:32,623 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:35291 2024-12-05T22:12:32,634 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:12:32,637 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:12:32,656 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:35291 connecting to ZooKeeper ensemble=127.0.0.1:52878 2024-12-05T22:12:32,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352910x0, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:12:32,698 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35291-0x10184afa89e0000 connected 2024-12-05T22:12:32,734 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:12:32,737 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:12:32,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:12:32,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35291 2024-12-05T22:12:32,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35291 2024-12-05T22:12:32,746 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35291 2024-12-05T22:12:32,746 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35291 2024-12-05T22:12:32,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35291 2024-12-05T22:12:32,755 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866, hbase.cluster.distributed=false 2024-12-05T22:12:32,839 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:12:32,839 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:12:32,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:12:32,840 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:12:32,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:12:32,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:12:32,843 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:12:32,847 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:12:32,848 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36117 2024-12-05T22:12:32,850 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T22:12:32,858 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T22:12:32,860 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:12:32,866 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:12:32,873 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:36117 connecting to ZooKeeper ensemble=127.0.0.1:52878 2024-12-05T22:12:32,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361170x0, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:12:32,879 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36117-0x10184afa89e0001 connected 2024-12-05T22:12:32,879 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:12:32,881 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:12:32,882 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:12:32,883 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36117 2024-12-05T22:12:32,883 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36117 2024-12-05T22:12:32,886 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36117 2024-12-05T22:12:32,887 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36117 2024-12-05T22:12:32,887 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36117 2024-12-05T22:12:32,890 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/d029b80fd32e,35291,1733436751952 2024-12-05T22:12:32,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:12:32,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:12:32,899 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d029b80fd32e,35291,1733436751952 2024-12-05T22:12:32,911 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d029b80fd32e:35291 2024-12-05T22:12:32,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:12:32,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:12:32,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:32,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:32,924 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:12:32,925 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d029b80fd32e,35291,1733436751952 from backup master directory 2024-12-05T22:12:32,926 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:12:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:12:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d029b80fd32e,35291,1733436751952 2024-12-05T22:12:32,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:12:32,932 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:12:32,932 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d029b80fd32e,35291,1733436751952 2024-12-05T22:12:32,935 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T22:12:32,937 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T22:12:33,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:12:33,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:12:33,031 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/hbase.id with ID: 24d05390-3a2f-4fef-b71a-97926b5a7776 2024-12-05T22:12:33,078 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:12:33,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:33,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:33,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:12:33,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:12:33,144 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:12:33,147 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T22:12:33,155 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:12:33,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:12:33,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:12:33,217 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store 2024-12-05T22:12:33,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:12:33,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:12:33,282 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T22:12:33,283 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:12:33,284 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:12:33,284 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:12:33,284 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:12:33,284 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:12:33,284 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:12:33,285 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:12:33,285 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:12:33,287 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/.initializing 2024-12-05T22:12:33,287 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/WALs/d029b80fd32e,35291,1733436751952 2024-12-05T22:12:33,316 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C35291%2C1733436751952, suffix=, logDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/WALs/d029b80fd32e,35291,1733436751952, archiveDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/oldWALs, maxLogs=10 2024-12-05T22:12:33,327 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C35291%2C1733436751952.1733436753324 2024-12-05T22:12:33,329 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(752): Using builder API via reflection for DFS file creation replicate flag. 2024-12-05T22:12:33,329 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(762): Using builder API via reflection for DFS file creation noLocalWrite flag. 2024-12-05T22:12:33,352 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/WALs/d029b80fd32e,35291,1733436751952/d029b80fd32e%2C35291%2C1733436751952.1733436753324 2024-12-05T22:12:33,361 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36661:36661),(127.0.0.1/127.0.0.1:36315:36315)] 2024-12-05T22:12:33,361 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:12:33,362 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:12:33,365 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:12:33,366 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:12:33,404 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:12:33,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T22:12:33,441 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:33,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:12:33,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:12:33,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T22:12:33,449 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:33,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:12:33,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:12:33,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T22:12:33,453 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:33,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:12:33,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:12:33,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T22:12:33,456 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:33,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:12:33,461 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:12:33,463 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:12:33,474 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T22:12:33,478 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:12:33,490 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:12:33,491 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=831980, jitterRate=0.05791833996772766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T22:12:33,497 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:12:33,498 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T22:12:33,536 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5475e002, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:12:33,578 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-05T22:12:33,591 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T22:12:33,591 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T22:12:33,593 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T22:12:33,595 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T22:12:33,599 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-12-05T22:12:33,600 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T22:12:33,632 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T22:12:33,643 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T22:12:33,645 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-05T22:12:33,647 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T22:12:33,648 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T22:12:33,650 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-05T22:12:33,652 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T22:12:33,655 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T22:12:33,657 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-05T22:12:33,658 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T22:12:33,661 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T22:12:33,673 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T22:12:33,675 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T22:12:33,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:12:33,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:12:33,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:33,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:33,681 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=d029b80fd32e,35291,1733436751952, sessionid=0x10184afa89e0000, setting cluster-up flag (Was=false) 2024-12-05T22:12:33,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:33,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:33,703 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T22:12:33,706 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,35291,1733436751952 2024-12-05T22:12:33,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:33,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:33,723 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T22:12:33,724 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,35291,1733436751952 2024-12-05T22:12:33,807 DEBUG [RS:0;d029b80fd32e:36117 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d029b80fd32e:36117 2024-12-05T22:12:33,808 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-05T22:12:33,809 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1008): ClusterId : 24d05390-3a2f-4fef-b71a-97926b5a7776 2024-12-05T22:12:33,812 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T22:12:33,815 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-05T22:12:33,816 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T22:12:33,817 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T22:12:33,818 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T22:12:33,820 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T22:12:33,821 DEBUG [RS:0;d029b80fd32e:36117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3f8f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:12:33,823 DEBUG [RS:0;d029b80fd32e:36117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e7c690e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:12:33,823 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d029b80fd32e,35291,1733436751952 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T22:12:33,827 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:12:33,827 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-05T22:12:33,827 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:12:33,827 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-05T22:12:33,827 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:12:33,827 DEBUG [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-05T22:12:33,827 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:12:33,828 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d029b80fd32e:0, corePoolSize=10, maxPoolSize=10 2024-12-05T22:12:33,828 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:33,828 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:12:33,828 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:33,830 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733436783830 2024-12-05T22:12:33,830 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(3073): reportForDuty to master=d029b80fd32e,35291,1733436751952 with isa=d029b80fd32e/172.17.0.2:36117, startcode=1733436752838 2024-12-05T22:12:33,831 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T22:12:33,833 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T22:12:33,834 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:12:33,834 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-05T22:12:33,837 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T22:12:33,838 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T22:12:33,838 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T22:12:33,838 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T22:12:33,839 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:33,839 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:33,839 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:12:33,843 DEBUG [RS:0;d029b80fd32e:36117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:12:33,845 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T22:12:33,846 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T22:12:33,847 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T22:12:33,850 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T22:12:33,850 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T22:12:33,851 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436753851,5,FailOnTimeoutGroup] 2024-12-05T22:12:33,852 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436753852,5,FailOnTimeoutGroup] 2024-12-05T22:12:33,852 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:33,852 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T22:12:33,854 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:33,854 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:33,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:12:33,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:12:33,863 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-05T22:12:33,863 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866 2024-12-05T22:12:33,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:12:33,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:12:33,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:12:33,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:12:33,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:12:33,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:33,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:12:33,890 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:12:33,893 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:12:33,894 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:33,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:12:33,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:12:33,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:12:33,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:33,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:12:33,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740 2024-12-05T22:12:33,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740 2024-12-05T22:12:33,906 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:12:33,909 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:12:33,917 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:12:33,919 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=862737, jitterRate=0.09702764451503754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:12:33,921 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50745, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:12:33,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:12:33,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:12:33,923 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:12:33,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:12:33,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:12:33,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:12:33,925 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:12:33,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:12:33,929 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35291 {}] master.ServerManager(332): Checking decommissioned status of RegionServer d029b80fd32e,36117,1733436752838 2024-12-05T22:12:33,929 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:12:33,929 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-05T22:12:33,931 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35291 {}] master.ServerManager(486): Registering regionserver=d029b80fd32e,36117,1733436752838 2024-12-05T22:12:33,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T22:12:33,946 DEBUG [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866 2024-12-05T22:12:33,946 DEBUG [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:37467 2024-12-05T22:12:33,947 DEBUG [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-05T22:12:33,947 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T22:12:33,951 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T22:12:33,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:12:33,953 DEBUG [RS:0;d029b80fd32e:36117 {}] zookeeper.ZKUtil(111): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d029b80fd32e,36117,1733436752838 2024-12-05T22:12:33,953 WARN [RS:0;d029b80fd32e:36117 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:12:33,954 INFO [RS:0;d029b80fd32e:36117 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:12:33,954 DEBUG [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838 2024-12-05T22:12:33,956 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d029b80fd32e,36117,1733436752838] 2024-12-05T22:12:33,969 DEBUG [RS:0;d029b80fd32e:36117 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-05T22:12:33,984 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T22:12:34,001 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T22:12:34,004 INFO [RS:0;d029b80fd32e:36117 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T22:12:34,005 INFO [RS:0;d029b80fd32e:36117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,005 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-05T22:12:34,014 INFO [RS:0;d029b80fd32e:36117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,015 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,015 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,015 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,015 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,016 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,016 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:12:34,016 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,016 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,017 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,017 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,017 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:12:34,017 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:12:34,017 DEBUG [RS:0;d029b80fd32e:36117 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:12:34,018 INFO [RS:0;d029b80fd32e:36117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,018 INFO [RS:0;d029b80fd32e:36117 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,019 INFO [RS:0;d029b80fd32e:36117 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,019 INFO [RS:0;d029b80fd32e:36117 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,019 INFO [RS:0;d029b80fd32e:36117 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,36117,1733436752838-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:12:34,040 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T22:12:34,042 INFO [RS:0;d029b80fd32e:36117 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,36117,1733436752838-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,067 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.Replication(204): d029b80fd32e,36117,1733436752838 started 2024-12-05T22:12:34,067 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1767): Serving as d029b80fd32e,36117,1733436752838, RpcServer on d029b80fd32e/172.17.0.2:36117, sessionid=0x10184afa89e0001 2024-12-05T22:12:34,068 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T22:12:34,068 DEBUG [RS:0;d029b80fd32e:36117 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d029b80fd32e,36117,1733436752838 2024-12-05T22:12:34,068 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,36117,1733436752838' 2024-12-05T22:12:34,068 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T22:12:34,069 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T22:12:34,070 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T22:12:34,070 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T22:12:34,070 DEBUG [RS:0;d029b80fd32e:36117 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d029b80fd32e,36117,1733436752838 2024-12-05T22:12:34,071 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,36117,1733436752838' 2024-12-05T22:12:34,071 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T22:12:34,071 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T22:12:34,072 DEBUG [RS:0;d029b80fd32e:36117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T22:12:34,072 INFO [RS:0;d029b80fd32e:36117 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T22:12:34,072 INFO [RS:0;d029b80fd32e:36117 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T22:12:34,101 WARN [d029b80fd32e:35291 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-05T22:12:34,182 INFO [RS:0;d029b80fd32e:36117 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C36117%2C1733436752838, suffix=, logDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838, archiveDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs, maxLogs=32 2024-12-05T22:12:34,186 INFO [RS:0;d029b80fd32e:36117 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436754185 2024-12-05T22:12:34,195 INFO [RS:0;d029b80fd32e:36117 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436754185 2024-12-05T22:12:34,195 DEBUG [RS:0;d029b80fd32e:36117 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36661:36661),(127.0.0.1/127.0.0.1:36315:36315)] 2024-12-05T22:12:34,353 DEBUG [d029b80fd32e:35291 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T22:12:34,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d029b80fd32e,36117,1733436752838 2024-12-05T22:12:34,363 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,36117,1733436752838, state=OPENING 2024-12-05T22:12:34,368 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T22:12:34,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:34,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:34,371 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:12:34,371 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:12:34,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=d029b80fd32e,36117,1733436752838}] 2024-12-05T22:12:34,547 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d029b80fd32e,36117,1733436752838 2024-12-05T22:12:34,548 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T22:12:34,552 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36526, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T22:12:34,564 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-05T22:12:34,565 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:12:34,568 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C36117%2C1733436752838.meta, suffix=.meta, logDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838, archiveDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs, maxLogs=32 2024-12-05T22:12:34,570 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.meta.1733436754570.meta 2024-12-05T22:12:34,580 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.meta.1733436754570.meta 2024-12-05T22:12:34,581 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36315:36315),(127.0.0.1/127.0.0.1:36661:36661)] 2024-12-05T22:12:34,581 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:12:34,582 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T22:12:34,645 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T22:12:34,650 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T22:12:34,656 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T22:12:34,656 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:12:34,656 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-05T22:12:34,656 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-05T22:12:34,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:12:34,663 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:12:34,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:34,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:12:34,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:12:34,666 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:12:34,666 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:34,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:12:34,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:12:34,669 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:12:34,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:34,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:12:34,672 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740 2024-12-05T22:12:34,675 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740 2024-12-05T22:12:34,678 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:12:34,681 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:12:34,683 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752842, jitterRate=-0.04271264374256134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:12:34,686 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:12:34,693 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733436754541 2024-12-05T22:12:34,704 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T22:12:34,704 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-05T22:12:34,705 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,36117,1733436752838 2024-12-05T22:12:34,707 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,36117,1733436752838, state=OPEN 2024-12-05T22:12:34,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:12:34,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:12:34,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:12:34,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:12:34,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T22:12:34,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=d029b80fd32e,36117,1733436752838 in 347 msec 2024-12-05T22:12:34,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T22:12:34,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 789 msec 2024-12-05T22:12:34,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 975 msec 2024-12-05T22:12:34,735 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733436754735, completionTime=-1 2024-12-05T22:12:34,735 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T22:12:34,735 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-05T22:12:34,775 DEBUG [hconnection-0x2e447fc7-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:12:34,777 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36534, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:12:34,788 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-05T22:12:34,788 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733436814788 2024-12-05T22:12:34,788 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733436874788 2024-12-05T22:12:34,788 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 52 msec 2024-12-05T22:12:34,814 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,35291,1733436751952-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,814 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,35291,1733436751952-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,814 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,35291,1733436751952-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,815 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d029b80fd32e:35291, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,816 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:34,821 DEBUG [master/d029b80fd32e:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-05T22:12:34,824 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-05T22:12:34,825 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:12:34,832 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-05T22:12:34,835 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:12:34,836 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:34,838 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:12:34,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:12:34,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:12:34,856 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4dca0a020ae6488f3eb687df1e1e0538, NAME => 'hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866 2024-12-05T22:12:34,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:12:34,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:12:34,878 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:12:34,879 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 4dca0a020ae6488f3eb687df1e1e0538, disabling compactions & flushes 2024-12-05T22:12:34,879 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:12:34,879 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:12:34,879 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. after waiting 0 ms 2024-12-05T22:12:34,879 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:12:34,880 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:12:34,880 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4dca0a020ae6488f3eb687df1e1e0538: 2024-12-05T22:12:34,887 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:12:34,894 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733436754889"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733436754889"}]},"ts":"1733436754889"} 2024-12-05T22:12:34,924 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:12:34,927 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:12:34,930 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436754927"}]},"ts":"1733436754927"} 2024-12-05T22:12:34,935 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-05T22:12:34,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=4dca0a020ae6488f3eb687df1e1e0538, ASSIGN}] 2024-12-05T22:12:34,943 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=4dca0a020ae6488f3eb687df1e1e0538, ASSIGN 2024-12-05T22:12:34,946 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=4dca0a020ae6488f3eb687df1e1e0538, ASSIGN; state=OFFLINE, location=d029b80fd32e,36117,1733436752838; forceNewPlan=false, retain=false 2024-12-05T22:12:35,097 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=4dca0a020ae6488f3eb687df1e1e0538, regionState=OPENING, regionLocation=d029b80fd32e,36117,1733436752838 2024-12-05T22:12:35,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 4dca0a020ae6488f3eb687df1e1e0538, server=d029b80fd32e,36117,1733436752838}] 2024-12-05T22:12:35,257 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d029b80fd32e,36117,1733436752838 2024-12-05T22:12:35,263 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:12:35,263 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 4dca0a020ae6488f3eb687df1e1e0538, NAME => 'hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:12:35,264 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 4dca0a020ae6488f3eb687df1e1e0538 2024-12-05T22:12:35,264 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:12:35,264 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 4dca0a020ae6488f3eb687df1e1e0538 2024-12-05T22:12:35,264 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 4dca0a020ae6488f3eb687df1e1e0538 2024-12-05T22:12:35,267 INFO [StoreOpener-4dca0a020ae6488f3eb687df1e1e0538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4dca0a020ae6488f3eb687df1e1e0538 2024-12-05T22:12:35,269 INFO [StoreOpener-4dca0a020ae6488f3eb687df1e1e0538-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dca0a020ae6488f3eb687df1e1e0538 columnFamilyName info 2024-12-05T22:12:35,269 DEBUG [StoreOpener-4dca0a020ae6488f3eb687df1e1e0538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:35,270 INFO [StoreOpener-4dca0a020ae6488f3eb687df1e1e0538-1 {}] regionserver.HStore(327): Store=4dca0a020ae6488f3eb687df1e1e0538/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:12:35,272 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/namespace/4dca0a020ae6488f3eb687df1e1e0538 2024-12-05T22:12:35,272 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/namespace/4dca0a020ae6488f3eb687df1e1e0538 2024-12-05T22:12:35,276 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 4dca0a020ae6488f3eb687df1e1e0538 2024-12-05T22:12:35,279 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/namespace/4dca0a020ae6488f3eb687df1e1e0538/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:12:35,280 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 4dca0a020ae6488f3eb687df1e1e0538; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=702313, jitterRate=-0.10696345567703247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:12:35,282 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 4dca0a020ae6488f3eb687df1e1e0538: 2024-12-05T22:12:35,284 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538., pid=6, masterSystemTime=1733436755257 2024-12-05T22:12:35,287 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:12:35,287 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:12:35,288 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=4dca0a020ae6488f3eb687df1e1e0538, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,36117,1733436752838 2024-12-05T22:12:35,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T22:12:35,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 4dca0a020ae6488f3eb687df1e1e0538, server=d029b80fd32e,36117,1733436752838 in 190 msec 2024-12-05T22:12:35,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T22:12:35,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=4dca0a020ae6488f3eb687df1e1e0538, ASSIGN in 355 msec 2024-12-05T22:12:35,302 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:12:35,302 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436755302"}]},"ts":"1733436755302"} 2024-12-05T22:12:35,305 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-05T22:12:35,309 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:12:35,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 483 msec 2024-12-05T22:12:35,335 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-05T22:12:35,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:35,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:12:35,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:12:35,375 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-05T22:12:35,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:12:35,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 27 msec 2024-12-05T22:12:35,409 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-05T22:12:35,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:12:35,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 19 msec 2024-12-05T22:12:35,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-05T22:12:35,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-05T22:12:35,450 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 2.518sec 2024-12-05T22:12:35,452 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T22:12:35,453 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T22:12:35,454 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T22:12:35,455 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T22:12:35,455 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T22:12:35,456 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,35291,1733436751952-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:12:35,456 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,35291,1733436751952-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T22:12:35,463 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-05T22:12:35,464 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T22:12:35,464 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,35291,1733436751952-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:12:35,508 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08f373c4 to 127.0.0.1:52878 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@59dcded7 2024-12-05T22:12:35,508 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-05T22:12:35,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586ea9a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:12:35,518 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T22:12:35,518 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T22:12:35,528 DEBUG [hconnection-0x4002cb39-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:12:35,535 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36536, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:12:35,546 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=d029b80fd32e,35291,1733436751952 2024-12-05T22:12:35,547 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:12:35,556 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-05T22:12:35,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T22:12:35,568 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48626, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T22:12:35,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35291 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-05T22:12:35,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35291 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-05T22:12:35,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35291 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:12:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35291 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-12-05T22:12:35,585 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:12:35,585 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:35,586 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:12:35,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35291 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 9 2024-12-05T22:12:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35291 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:12:35,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741837_1013 (size=389) 2024-12-05T22:12:35,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741837_1013 (size=389) 2024-12-05T22:12:35,652 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 36566434b3ab5e2872b6ee8cdb0c20a7, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866 2024-12-05T22:12:35,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741838_1014 (size=72) 2024-12-05T22:12:35,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741838_1014 (size=72) 2024-12-05T22:12:36,066 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:12:36,066 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing 36566434b3ab5e2872b6ee8cdb0c20a7, disabling compactions & flushes 2024-12-05T22:12:36,066 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:12:36,066 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:12:36,066 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. after waiting 0 ms 2024-12-05T22:12:36,066 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:12:36,067 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:12:36,067 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 36566434b3ab5e2872b6ee8cdb0c20a7: 2024-12-05T22:12:36,069 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:12:36,069 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733436756069"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733436756069"}]},"ts":"1733436756069"} 2024-12-05T22:12:36,072 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:12:36,074 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:12:36,074 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436756074"}]},"ts":"1733436756074"} 2024-12-05T22:12:36,077 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-12-05T22:12:36,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=36566434b3ab5e2872b6ee8cdb0c20a7, ASSIGN}] 2024-12-05T22:12:36,085 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=36566434b3ab5e2872b6ee8cdb0c20a7, ASSIGN 2024-12-05T22:12:36,086 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=36566434b3ab5e2872b6ee8cdb0c20a7, ASSIGN; state=OFFLINE, location=d029b80fd32e,36117,1733436752838; forceNewPlan=false, retain=false 2024-12-05T22:12:36,237 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=36566434b3ab5e2872b6ee8cdb0c20a7, regionState=OPENING, regionLocation=d029b80fd32e,36117,1733436752838 2024-12-05T22:12:36,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 36566434b3ab5e2872b6ee8cdb0c20a7, server=d029b80fd32e,36117,1733436752838}] 2024-12-05T22:12:36,394 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d029b80fd32e,36117,1733436752838 2024-12-05T22:12:36,402 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:12:36,402 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 36566434b3ab5e2872b6ee8cdb0c20a7, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:12:36,403 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:12:36,403 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:12:36,403 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:12:36,403 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:12:36,407 INFO [StoreOpener-36566434b3ab5e2872b6ee8cdb0c20a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:12:36,410 INFO [StoreOpener-36566434b3ab5e2872b6ee8cdb0c20a7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36566434b3ab5e2872b6ee8cdb0c20a7 columnFamilyName info 2024-12-05T22:12:36,410 DEBUG [StoreOpener-36566434b3ab5e2872b6ee8cdb0c20a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:12:36,412 INFO [StoreOpener-36566434b3ab5e2872b6ee8cdb0c20a7-1 {}] regionserver.HStore(327): Store=36566434b3ab5e2872b6ee8cdb0c20a7/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:12:36,413 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:12:36,414 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:12:36,417 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:12:36,421 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:12:36,421 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 36566434b3ab5e2872b6ee8cdb0c20a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=742558, jitterRate=-0.05578938126564026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:12:36,423 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 36566434b3ab5e2872b6ee8cdb0c20a7: 2024-12-05T22:12:36,424 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7., pid=11, masterSystemTime=1733436756394 2024-12-05T22:12:36,428 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:12:36,428 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:12:36,429 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=36566434b3ab5e2872b6ee8cdb0c20a7, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,36117,1733436752838 2024-12-05T22:12:36,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-05T22:12:36,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 36566434b3ab5e2872b6ee8cdb0c20a7, server=d029b80fd32e,36117,1733436752838 in 191 msec 2024-12-05T22:12:36,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T22:12:36,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=36566434b3ab5e2872b6ee8cdb0c20a7, ASSIGN in 353 msec 2024-12-05T22:12:36,442 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:12:36,442 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436756442"}]},"ts":"1733436756442"} 2024-12-05T22:12:36,445 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-12-05T22:12:36,448 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:12:36,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 868 msec 2024-12-05T22:12:40,193 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T22:12:40,248 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T22:12:40,250 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-05T22:12:40,251 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-12-05T22:12:42,542 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T22:12:42,542 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T22:12:42,544 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-05T22:12:42,544 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-05T22:12:42,546 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-05T22:12:42,546 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-12-05T22:12:42,547 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T22:12:42,547 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T22:12:42,548 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T22:12:42,548 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T22:12:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35291 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:12:45,599 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling, procId: 9 completed 2024-12-05T22:12:45,604 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-12-05T22:12:45,605 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:12:45,605 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436765605 2024-12-05T22:12:45,616 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436754185 with entries=4, filesize=947 B; new WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436765605 2024-12-05T22:12:45,617 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36661:36661),(127.0.0.1/127.0.0.1:36315:36315)] 2024-12-05T22:12:45,617 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436754185 is not closed yet, will try archiving it next time 2024-12-05T22:12:45,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741833_1009 (size=955) 2024-12-05T22:12:45,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741833_1009 (size=955) 2024-12-05T22:12:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36117 {}] regionserver.HRegion(8581): Flush requested on 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:12:57,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 36566434b3ab5e2872b6ee8cdb0c20a7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:12:57,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/08c6fb7659914ac4b61e896b04bc2829 is 1080, key is row0001/info:/1733436765622/Put/seqid=0 2024-12-05T22:12:57,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741840_1016 (size=12509) 2024-12-05T22:12:57,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741840_1016 (size=12509) 2024-12-05T22:12:57,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/08c6fb7659914ac4b61e896b04bc2829 2024-12-05T22:12:57,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/08c6fb7659914ac4b61e896b04bc2829 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/08c6fb7659914ac4b61e896b04bc2829 2024-12-05T22:12:57,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/08c6fb7659914ac4b61e896b04bc2829, entries=7, sequenceid=11, filesize=12.2 K 2024-12-05T22:12:57,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 36566434b3ab5e2872b6ee8cdb0c20a7 in 221ms, sequenceid=11, compaction requested=false 2024-12-05T22:12:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 36566434b3ab5e2872b6ee8cdb0c20a7: 2024-12-05T22:13:01,289 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:13:04,825 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:13:04,828 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49690, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:13:05,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436785658 2024-12-05T22:13:05,868 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 207 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:05,869 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436765605 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436785658 2024-12-05T22:13:05,870 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36661:36661),(127.0.0.1/127.0.0.1:36315:36315)] 2024-12-05T22:13:05,870 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436765605 is not closed yet, will try archiving it next time 2024-12-05T22:13:05,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741839_1015 (size=12399) 2024-12-05T22:13:05,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741839_1015 (size=12399) 2024-12-05T22:13:06,073 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:08,276 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:10,479 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:12,683 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36117 {}] regionserver.HRegion(8581): Flush requested on 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:13:12,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 36566434b3ab5e2872b6ee8cdb0c20a7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:13:12,885 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:12,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/817fce14906f4935b9d2ee8a74e6fe7d is 1080, key is row0008/info:/1733436779649/Put/seqid=0 2024-12-05T22:13:12,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741842_1018 (size=12509) 2024-12-05T22:13:12,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741842_1018 (size=12509) 2024-12-05T22:13:12,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/817fce14906f4935b9d2ee8a74e6fe7d 2024-12-05T22:13:12,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/817fce14906f4935b9d2ee8a74e6fe7d as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/817fce14906f4935b9d2ee8a74e6fe7d 2024-12-05T22:13:12,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/817fce14906f4935b9d2ee8a74e6fe7d, entries=7, sequenceid=21, filesize=12.2 K 2024-12-05T22:13:13,119 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:13,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 36566434b3ab5e2872b6ee8cdb0c20a7 in 436ms, sequenceid=21, compaction requested=false 2024-12-05T22:13:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 36566434b3ab5e2872b6ee8cdb0c20a7: 2024-12-05T22:13:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=24.4 K, sizeToCheck=16.0 K 2024-12-05T22:13:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:13:13,121 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/08c6fb7659914ac4b61e896b04bc2829 because midkey is the same as first or last row 2024-12-05T22:13:14,889 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:16,061 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T22:13:16,061 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T22:13:17,098 WARN [sync.1 {}] wal.AbstractFSWAL(1346): Requesting log roll because we exceeded slow sync threshold; count=7, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:17,100 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:17,100 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C36117%2C1733436752838:(num 1733436785658) roll requested 2024-12-05T22:13:17,101 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436797101 2024-12-05T22:13:17,313 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:17,514 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:17,514 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436785658 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436797101 2024-12-05T22:13:17,514 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36661:36661),(127.0.0.1/127.0.0.1:36315:36315)] 2024-12-05T22:13:17,515 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436785658 is not closed yet, will try archiving it next time 2024-12-05T22:13:17,516 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436765605 to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs/d029b80fd32e%2C36117%2C1733436752838.1733436765605 2024-12-05T22:13:17,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741841_1017 (size=7739) 2024-12-05T22:13:17,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741841_1017 (size=7739) 2024-12-05T22:13:19,303 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:21,403 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 36566434b3ab5e2872b6ee8cdb0c20a7, had cached 0 bytes from a total of 25018 2024-12-05T22:13:21,507 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:23,711 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:25,917 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:27,919 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T22:13:27,920 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436807920 2024-12-05T22:13:31,289 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:13:32,930 INFO [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:32,930 WARN [Time-limited test {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:32,930 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C36117%2C1733436752838:(num 1733436807920) roll requested 2024-12-05T22:13:34,947 DEBUG [master/d029b80fd32e:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 4dca0a020ae6488f3eb687df1e1e0538 changed from -1.0 to 0.0, refreshing cache 2024-12-05T22:13:37,931 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:37,931 WARN [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:37,932 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436797101 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436807920 2024-12-05T22:13:37,933 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36661:36661),(127.0.0.1/127.0.0.1:36315:36315)] 2024-12-05T22:13:37,933 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436797101 is not closed yet, will try archiving it next time 2024-12-05T22:13:37,933 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436817933 2024-12-05T22:13:37,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741843_1019 (size=4753) 2024-12-05T22:13:37,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741843_1019 (size=4753) 2024-12-05T22:13:42,936 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:42,936 WARN [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36117 {}] regionserver.HRegion(8581): Flush requested on 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:13:42,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 36566434b3ab5e2872b6ee8cdb0c20a7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:13:42,944 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5008 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:42,944 WARN [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5008 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:44,938 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T22:13:47,939 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:47,939 WARN [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:47,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/700e839d9eb34c608fb3171aa02db8e5 is 1080, key is row0015/info:/1733436794687/Put/seqid=0 2024-12-05T22:13:47,945 INFO [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:47,945 WARN [sync.0 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:13:47,946 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436807920 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436817933 2024-12-05T22:13:47,946 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36315:36315),(127.0.0.1/127.0.0.1:36661:36661)] 2024-12-05T22:13:47,946 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436807920 is not closed yet, will try archiving it next time 2024-12-05T22:13:47,947 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C36117%2C1733436752838:(num 1733436817933) roll requested 2024-12-05T22:13:47,947 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436827947 2024-12-05T22:13:47,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741844_1020 (size=1569) 2024-12-05T22:13:47,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741844_1020 (size=1569) 2024-12-05T22:13:47,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741846_1022 (size=12509) 2024-12-05T22:13:47,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741846_1022 (size=12509) 2024-12-05T22:13:47,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/700e839d9eb34c608fb3171aa02db8e5 2024-12-05T22:13:47,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/700e839d9eb34c608fb3171aa02db8e5 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/700e839d9eb34c608fb3171aa02db8e5 2024-12-05T22:13:47,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/700e839d9eb34c608fb3171aa02db8e5, entries=7, sequenceid=31, filesize=12.2 K 2024-12-05T22:13:52,960 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK], DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK]] 2024-12-05T22:13:52,960 WARN [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK], DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK]] 2024-12-05T22:13:52,976 INFO [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK], DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK]] 2024-12-05T22:13:52,976 WARN [sync.1 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK], DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK]] 2024-12-05T22:13:52,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 36566434b3ab5e2872b6ee8cdb0c20a7 in 10040ms, sequenceid=31, compaction requested=true 2024-12-05T22:13:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 36566434b3ab5e2872b6ee8cdb0c20a7: 2024-12-05T22:13:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=36.6 K, sizeToCheck=16.0 K 2024-12-05T22:13:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:13:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/08c6fb7659914ac4b61e896b04bc2829 because midkey is the same as first or last row 2024-12-05T22:13:52,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36566434b3ab5e2872b6ee8cdb0c20a7:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:13:52,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:13:52,978 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:13:52,981 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:13:52,983 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.HStore(1540): 36566434b3ab5e2872b6ee8cdb0c20a7/info is initiating minor compaction (all files) 2024-12-05T22:13:52,983 INFO [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 36566434b3ab5e2872b6ee8cdb0c20a7/info in TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:13:52,984 INFO [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/08c6fb7659914ac4b61e896b04bc2829, hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/817fce14906f4935b9d2ee8a74e6fe7d, hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/700e839d9eb34c608fb3171aa02db8e5] into tmpdir=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp, totalSize=36.6 K 2024-12-05T22:13:52,985 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] compactions.Compactor(224): Compacting 08c6fb7659914ac4b61e896b04bc2829, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733436765622 2024-12-05T22:13:52,986 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] compactions.Compactor(224): Compacting 817fce14906f4935b9d2ee8a74e6fe7d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733436779649 2024-12-05T22:13:52,987 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] compactions.Compactor(224): Compacting 700e839d9eb34c608fb3171aa02db8e5, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733436794687 2024-12-05T22:13:53,012 INFO [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36566434b3ab5e2872b6ee8cdb0c20a7#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:13:53,014 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/469be5bfceb5422abc192da3b6b9c946 is 1080, key is row0001/info:/1733436765622/Put/seqid=0 2024-12-05T22:13:53,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741848_1024 (size=27710) 2024-12-05T22:13:53,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741848_1024 (size=27710) 2024-12-05T22:13:53,031 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/469be5bfceb5422abc192da3b6b9c946 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/469be5bfceb5422abc192da3b6b9c946 2024-12-05T22:13:57,960 INFO [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK], DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK]] 2024-12-05T22:13:57,961 WARN [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK], DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK]] 2024-12-05T22:13:57,962 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436817933 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436827947 2024-12-05T22:13:57,962 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36661:36661),(127.0.0.1/127.0.0.1:36315:36315)] 2024-12-05T22:13:57,962 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436817933 is not closed yet, will try archiving it next time 2024-12-05T22:13:57,962 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C36117%2C1733436752838:(num 1733436827947) roll requested 2024-12-05T22:13:57,962 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436785658 to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs/d029b80fd32e%2C36117%2C1733436752838.1733436785658 2024-12-05T22:13:57,962 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436837962 2024-12-05T22:13:57,965 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436797101 to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs/d029b80fd32e%2C36117%2C1733436752838.1733436797101 2024-12-05T22:13:57,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741845_1021 (size=438) 2024-12-05T22:13:57,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741845_1021 (size=438) 2024-12-05T22:13:57,968 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436807920 to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs/d029b80fd32e%2C36117%2C1733436752838.1733436807920 2024-12-05T22:13:57,969 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436817933 to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs/d029b80fd32e%2C36117%2C1733436752838.1733436817933 2024-12-05T22:14:01,289 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:14:02,963 INFO [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:14:02,963 WARN [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:14:02,965 INFO [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 36566434b3ab5e2872b6ee8cdb0c20a7/info of 36566434b3ab5e2872b6ee8cdb0c20a7 into 469be5bfceb5422abc192da3b6b9c946(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 9sec to execute. 2024-12-05T22:14:02,965 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 36566434b3ab5e2872b6ee8cdb0c20a7: 2024-12-05T22:14:02,965 INFO [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7., storeName=36566434b3ab5e2872b6ee8cdb0c20a7/info, priority=13, startTime=1733436832978; duration=9sec 2024-12-05T22:14:02,965 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=27.1 K, sizeToCheck=16.0 K 2024-12-05T22:14:02,965 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:14:02,965 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/469be5bfceb5422abc192da3b6b9c946 because midkey is the same as first or last row 2024-12-05T22:14:02,965 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:14:02,965 DEBUG [RS:0;d029b80fd32e:36117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36566434b3ab5e2872b6ee8cdb0c20a7:info 2024-12-05T22:14:02,978 INFO [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:14:02,978 WARN [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44383,DS-ddf4de9f-ba80-4223-9f01-36d89caf09c2,DISK], DatanodeInfoWithStorage[127.0.0.1:43775,DS-8cb37b21-2bb1-4287-8634-382232d8217e,DISK]] 2024-12-05T22:14:02,978 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436827947 with entries=1, filesize=531 B; new WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436837962 2024-12-05T22:14:02,979 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36661:36661),(127.0.0.1/127.0.0.1:36315:36315)] 2024-12-05T22:14:02,979 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436827947 is not closed yet, will try archiving it next time 2024-12-05T22:14:02,979 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436842979 2024-12-05T22:14:02,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741847_1023 (size=539) 2024-12-05T22:14:02,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741847_1023 (size=539) 2024-12-05T22:14:02,982 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436827947 to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs/d029b80fd32e%2C36117%2C1733436752838.1733436827947 2024-12-05T22:14:02,987 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436837962 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436842979 2024-12-05T22:14:02,988 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36315:36315),(127.0.0.1/127.0.0.1:36661:36661)] 2024-12-05T22:14:02,988 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436837962 is not closed yet, will try archiving it next time 2024-12-05T22:14:02,988 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C36117%2C1733436752838:(num 1733436842979) roll requested 2024-12-05T22:14:02,988 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C36117%2C1733436752838.1733436842988 2024-12-05T22:14:02,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741849_1025 (size=1258) 2024-12-05T22:14:02,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741849_1025 (size=1258) 2024-12-05T22:14:02,998 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436842979 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436842988 2024-12-05T22:14:02,998 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36315:36315),(127.0.0.1/127.0.0.1:36661:36661)] 2024-12-05T22:14:02,998 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436842979 is not closed yet, will try archiving it next time 2024-12-05T22:14:02,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741850_1026 (size=93) 2024-12-05T22:14:03,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741850_1026 (size=93) 2024-12-05T22:14:03,000 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838/d029b80fd32e%2C36117%2C1733436752838.1733436842979 to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs/d029b80fd32e%2C36117%2C1733436752838.1733436842979 2024-12-05T22:14:06,404 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 36566434b3ab5e2872b6ee8cdb0c20a7, had cached 0 bytes from a total of 27710 2024-12-05T22:14:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36117 {}] regionserver.HRegion(8581): Flush requested on 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:14:14,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 36566434b3ab5e2872b6ee8cdb0c20a7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:14:15,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/ecf98848ceb44cec95f303658a372169 is 1080, key is row0022/info:/1733436842980/Put/seqid=0 2024-12-05T22:14:15,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741852_1028 (size=12509) 2024-12-05T22:14:15,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741852_1028 (size=12509) 2024-12-05T22:14:15,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/ecf98848ceb44cec95f303658a372169 2024-12-05T22:14:15,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/ecf98848ceb44cec95f303658a372169 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/ecf98848ceb44cec95f303658a372169 2024-12-05T22:14:15,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/ecf98848ceb44cec95f303658a372169, entries=7, sequenceid=42, filesize=12.2 K 2024-12-05T22:14:15,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 36566434b3ab5e2872b6ee8cdb0c20a7 in 35ms, sequenceid=42, compaction requested=false 2024-12-05T22:14:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 36566434b3ab5e2872b6ee8cdb0c20a7: 2024-12-05T22:14:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=39.3 K, sizeToCheck=16.0 K 2024-12-05T22:14:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:14:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/469be5bfceb5422abc192da3b6b9c946 because midkey is the same as first or last row 2024-12-05T22:14:23,007 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-05T22:14:23,007 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-05T22:14:23,007 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08f373c4 to 127.0.0.1:52878 2024-12-05T22:14:23,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:14:23,008 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T22:14:23,008 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1179894836, stopped=false 2024-12-05T22:14:23,009 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=d029b80fd32e,35291,1733436751952 2024-12-05T22:14:23,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:14:23,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:14:23,011 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-05T22:14:23,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:23,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:23,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:14:23,012 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,36117,1733436752838' ***** 2024-12-05T22:14:23,012 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-05T22:14:23,012 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:14:23,012 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:14:23,012 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T22:14:23,012 INFO [RS:0;d029b80fd32e:36117 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T22:14:23,013 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-05T22:14:23,013 INFO [RS:0;d029b80fd32e:36117 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T22:14:23,013 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(3579): Received CLOSE for 36566434b3ab5e2872b6ee8cdb0c20a7 2024-12-05T22:14:23,013 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(3579): Received CLOSE for 4dca0a020ae6488f3eb687df1e1e0538 2024-12-05T22:14:23,013 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,36117,1733436752838 2024-12-05T22:14:23,013 DEBUG [RS:0;d029b80fd32e:36117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:14:23,014 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T22:14:23,014 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T22:14:23,014 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T22:14:23,014 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-05T22:14:23,014 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 36566434b3ab5e2872b6ee8cdb0c20a7, disabling compactions & flushes 2024-12-05T22:14:23,014 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:14:23,014 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-05T22:14:23,014 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:14:23,014 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. after waiting 0 ms 2024-12-05T22:14:23,014 DEBUG [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1603): Online Regions={36566434b3ab5e2872b6ee8cdb0c20a7=TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7., 1588230740=hbase:meta,,1.1588230740, 4dca0a020ae6488f3eb687df1e1e0538=hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538.} 2024-12-05T22:14:23,014 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:14:23,014 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:14:23,014 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:14:23,014 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:14:23,014 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:14:23,014 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 36566434b3ab5e2872b6ee8cdb0c20a7 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-12-05T22:14:23,014 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:14:23,014 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.81 KB heapSize=5.32 KB 2024-12-05T22:14:23,014 DEBUG [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 36566434b3ab5e2872b6ee8cdb0c20a7, 4dca0a020ae6488f3eb687df1e1e0538 2024-12-05T22:14:23,019 INFO [regionserver/d029b80fd32e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T22:14:23,019 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/500ed246c47c4a2496b30a10890a6449 is 1080, key is row0029/info:/1733436857000/Put/seqid=0 2024-12-05T22:14:23,020 INFO [regionserver/d029b80fd32e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T22:14:23,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741853_1029 (size=8193) 2024-12-05T22:14:23,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741853_1029 (size=8193) 2024-12-05T22:14:23,031 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/500ed246c47c4a2496b30a10890a6449 2024-12-05T22:14:23,035 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/.tmp/info/833b7abbf1f8480b94a9cf3a47e28406 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7./info:regioninfo/1733436756429/Put/seqid=0 2024-12-05T22:14:23,040 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/.tmp/info/500ed246c47c4a2496b30a10890a6449 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/500ed246c47c4a2496b30a10890a6449 2024-12-05T22:14:23,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741854_1030 (size=8172) 2024-12-05T22:14:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741854_1030 (size=8172) 2024-12-05T22:14:23,042 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.59 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/.tmp/info/833b7abbf1f8480b94a9cf3a47e28406 2024-12-05T22:14:23,049 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/500ed246c47c4a2496b30a10890a6449, entries=3, sequenceid=48, filesize=8.0 K 2024-12-05T22:14:23,050 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 36566434b3ab5e2872b6ee8cdb0c20a7 in 36ms, sequenceid=48, compaction requested=true 2024-12-05T22:14:23,051 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/08c6fb7659914ac4b61e896b04bc2829, hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/817fce14906f4935b9d2ee8a74e6fe7d, hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/700e839d9eb34c608fb3171aa02db8e5] to archive 2024-12-05T22:14:23,053 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T22:14:23,057 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/08c6fb7659914ac4b61e896b04bc2829 to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/archive/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/08c6fb7659914ac4b61e896b04bc2829 2024-12-05T22:14:23,059 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/817fce14906f4935b9d2ee8a74e6fe7d to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/archive/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/817fce14906f4935b9d2ee8a74e6fe7d 2024-12-05T22:14:23,061 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/700e839d9eb34c608fb3171aa02db8e5 to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/archive/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/info/700e839d9eb34c608fb3171aa02db8e5 2024-12-05T22:14:23,066 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/.tmp/table/13e51d469c024fc7beb79e03fa8f66a0 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733436756442/Put/seqid=0 2024-12-05T22:14:23,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741855_1031 (size=5452) 2024-12-05T22:14:23,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741855_1031 (size=5452) 2024-12-05T22:14:23,073 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=232 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/.tmp/table/13e51d469c024fc7beb79e03fa8f66a0 2024-12-05T22:14:23,081 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/default/TestLogRolling-testSlowSyncLogRolling/36566434b3ab5e2872b6ee8cdb0c20a7/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-12-05T22:14:23,082 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/.tmp/info/833b7abbf1f8480b94a9cf3a47e28406 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/info/833b7abbf1f8480b94a9cf3a47e28406 2024-12-05T22:14:23,084 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:14:23,084 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 36566434b3ab5e2872b6ee8cdb0c20a7: 2024-12-05T22:14:23,084 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733436755575.36566434b3ab5e2872b6ee8cdb0c20a7. 2024-12-05T22:14:23,084 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4dca0a020ae6488f3eb687df1e1e0538, disabling compactions & flushes 2024-12-05T22:14:23,084 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:14:23,084 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:14:23,084 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. after waiting 0 ms 2024-12-05T22:14:23,084 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:14:23,084 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4dca0a020ae6488f3eb687df1e1e0538 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-05T22:14:23,090 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/info/833b7abbf1f8480b94a9cf3a47e28406, entries=20, sequenceid=14, filesize=8.0 K 2024-12-05T22:14:23,091 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/.tmp/table/13e51d469c024fc7beb79e03fa8f66a0 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/table/13e51d469c024fc7beb79e03fa8f66a0 2024-12-05T22:14:23,098 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/table/13e51d469c024fc7beb79e03fa8f66a0, entries=4, sequenceid=14, filesize=5.3 K 2024-12-05T22:14:23,099 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~2.81 KB/2882, heapSize ~5.04 KB/5160, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=14, compaction requested=false 2024-12-05T22:14:23,101 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/namespace/4dca0a020ae6488f3eb687df1e1e0538/.tmp/info/4c34eba6317a41fbb7afb6ecee47d0d3 is 45, key is default/info:d/1733436755386/Put/seqid=0 2024-12-05T22:14:23,105 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1 2024-12-05T22:14:23,106 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T22:14:23,106 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:14:23,106 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:14:23,106 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T22:14:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741856_1032 (size=5037) 2024-12-05T22:14:23,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741856_1032 (size=5037) 2024-12-05T22:14:23,111 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/namespace/4dca0a020ae6488f3eb687df1e1e0538/.tmp/info/4c34eba6317a41fbb7afb6ecee47d0d3 2024-12-05T22:14:23,118 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/namespace/4dca0a020ae6488f3eb687df1e1e0538/.tmp/info/4c34eba6317a41fbb7afb6ecee47d0d3 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/namespace/4dca0a020ae6488f3eb687df1e1e0538/info/4c34eba6317a41fbb7afb6ecee47d0d3 2024-12-05T22:14:23,125 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/namespace/4dca0a020ae6488f3eb687df1e1e0538/info/4c34eba6317a41fbb7afb6ecee47d0d3, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T22:14:23,126 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 4dca0a020ae6488f3eb687df1e1e0538 in 41ms, sequenceid=6, compaction requested=false 2024-12-05T22:14:23,130 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/data/hbase/namespace/4dca0a020ae6488f3eb687df1e1e0538/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T22:14:23,131 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:14:23,131 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4dca0a020ae6488f3eb687df1e1e0538: 2024-12-05T22:14:23,131 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733436754825.4dca0a020ae6488f3eb687df1e1e0538. 2024-12-05T22:14:23,215 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,36117,1733436752838; all regions closed. 2024-12-05T22:14:23,216 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838 2024-12-05T22:14:23,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741834_1010 (size=4330) 2024-12-05T22:14:23,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741834_1010 (size=4330) 2024-12-05T22:14:23,223 DEBUG [RS:0;d029b80fd32e:36117 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs 2024-12-05T22:14:23,224 INFO [RS:0;d029b80fd32e:36117 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog d029b80fd32e%2C36117%2C1733436752838.meta:.meta(num 1733436754570) 2024-12-05T22:14:23,224 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/WALs/d029b80fd32e,36117,1733436752838 2024-12-05T22:14:23,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741851_1027 (size=13066) 2024-12-05T22:14:23,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741851_1027 (size=13066) 2024-12-05T22:14:23,232 DEBUG [RS:0;d029b80fd32e:36117 {}] wal.AbstractFSWAL(1071): Moved 3 WAL file(s) to /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/oldWALs 2024-12-05T22:14:23,232 INFO [RS:0;d029b80fd32e:36117 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog d029b80fd32e%2C36117%2C1733436752838:(num 1733436842988) 2024-12-05T22:14:23,232 DEBUG [RS:0;d029b80fd32e:36117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:14:23,232 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:14:23,232 INFO [RS:0;d029b80fd32e:36117 {}] hbase.ChoreService(370): Chore service for: regionserver/d029b80fd32e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-05T22:14:23,233 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:14:23,233 INFO [RS:0;d029b80fd32e:36117 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36117 2024-12-05T22:14:23,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d029b80fd32e,36117,1733436752838 2024-12-05T22:14:23,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:14:23,238 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d029b80fd32e,36117,1733436752838] 2024-12-05T22:14:23,239 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing d029b80fd32e,36117,1733436752838; numProcessing=1 2024-12-05T22:14:23,240 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/d029b80fd32e,36117,1733436752838 already deleted, retry=false 2024-12-05T22:14:23,240 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; d029b80fd32e,36117,1733436752838 expired; onlineServers=0 2024-12-05T22:14:23,240 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,35291,1733436751952' ***** 2024-12-05T22:14:23,240 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T22:14:23,240 DEBUG [M:0;d029b80fd32e:35291 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efc055c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:14:23,240 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,35291,1733436751952 2024-12-05T22:14:23,240 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,35291,1733436751952; all regions closed. 2024-12-05T22:14:23,240 DEBUG [M:0;d029b80fd32e:35291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:14:23,240 DEBUG [M:0;d029b80fd32e:35291 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T22:14:23,241 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T22:14:23,241 DEBUG [M:0;d029b80fd32e:35291 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T22:14:23,241 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436753851 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436753851,5,FailOnTimeoutGroup] 2024-12-05T22:14:23,241 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436753852 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436753852,5,FailOnTimeoutGroup] 2024-12-05T22:14:23,241 INFO [M:0;d029b80fd32e:35291 {}] hbase.ChoreService(370): Chore service for: master/d029b80fd32e:0 had [] on shutdown 2024-12-05T22:14:23,241 DEBUG [M:0;d029b80fd32e:35291 {}] master.HMaster(1733): Stopping service threads 2024-12-05T22:14:23,241 INFO [M:0;d029b80fd32e:35291 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T22:14:23,242 INFO [M:0;d029b80fd32e:35291 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T22:14:23,242 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T22:14:23,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T22:14:23,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:23,243 DEBUG [M:0;d029b80fd32e:35291 {}] zookeeper.ZKUtil(347): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T22:14:23,243 WARN [M:0;d029b80fd32e:35291 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T22:14:23,243 INFO [M:0;d029b80fd32e:35291 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-05T22:14:23,243 INFO [M:0;d029b80fd32e:35291 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T22:14:23,243 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:14:23,243 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:14:23,244 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:14:23,244 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:14:23,244 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:14:23,244 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:14:23,244 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.20 KB heapSize=50.12 KB 2024-12-05T22:14:23,269 DEBUG [M:0;d029b80fd32e:35291 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd82acf75b3d48efaafb93405b1a8492 is 82, key is hbase:meta,,1/info:regioninfo/1733436754705/Put/seqid=0 2024-12-05T22:14:23,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741857_1033 (size=5672) 2024-12-05T22:14:23,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741857_1033 (size=5672) 2024-12-05T22:14:23,276 INFO [M:0;d029b80fd32e:35291 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd82acf75b3d48efaafb93405b1a8492 2024-12-05T22:14:23,299 DEBUG [M:0;d029b80fd32e:35291 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77f591c327b043c2a8823323721c734c is 765, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733436756450/Put/seqid=0 2024-12-05T22:14:23,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741858_1034 (size=6425) 2024-12-05T22:14:23,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741858_1034 (size=6425) 2024-12-05T22:14:23,305 INFO [M:0;d029b80fd32e:35291 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.59 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77f591c327b043c2a8823323721c734c 2024-12-05T22:14:23,313 INFO [M:0;d029b80fd32e:35291 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 77f591c327b043c2a8823323721c734c 2024-12-05T22:14:23,332 DEBUG [M:0;d029b80fd32e:35291 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5804983d88294753b6d83d9e15539dc3 is 69, key is d029b80fd32e,36117,1733436752838/rs:state/1733436753933/Put/seqid=0 2024-12-05T22:14:23,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:14:23,338 INFO [RS:0;d029b80fd32e:36117 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,36117,1733436752838; zookeeper connection closed. 2024-12-05T22:14:23,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36117-0x10184afa89e0001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:14:23,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741859_1035 (size=5156) 2024-12-05T22:14:23,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741859_1035 (size=5156) 2024-12-05T22:14:23,339 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@24a172fc {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@24a172fc 2024-12-05T22:14:23,339 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T22:14:23,340 INFO [M:0;d029b80fd32e:35291 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5804983d88294753b6d83d9e15539dc3 2024-12-05T22:14:23,363 DEBUG [M:0;d029b80fd32e:35291 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6cf235ea8185447c9d66450e6eaecf62 is 52, key is load_balancer_on/state:d/1733436755553/Put/seqid=0 2024-12-05T22:14:23,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741860_1036 (size=5056) 2024-12-05T22:14:23,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741860_1036 (size=5056) 2024-12-05T22:14:23,370 INFO [M:0;d029b80fd32e:35291 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6cf235ea8185447c9d66450e6eaecf62 2024-12-05T22:14:23,378 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd82acf75b3d48efaafb93405b1a8492 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd82acf75b3d48efaafb93405b1a8492 2024-12-05T22:14:23,384 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd82acf75b3d48efaafb93405b1a8492, entries=8, sequenceid=104, filesize=5.5 K 2024-12-05T22:14:23,386 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77f591c327b043c2a8823323721c734c as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/77f591c327b043c2a8823323721c734c 2024-12-05T22:14:23,392 INFO [M:0;d029b80fd32e:35291 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 77f591c327b043c2a8823323721c734c 2024-12-05T22:14:23,392 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/77f591c327b043c2a8823323721c734c, entries=11, sequenceid=104, filesize=6.3 K 2024-12-05T22:14:23,394 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5804983d88294753b6d83d9e15539dc3 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5804983d88294753b6d83d9e15539dc3 2024-12-05T22:14:23,399 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5804983d88294753b6d83d9e15539dc3, entries=1, sequenceid=104, filesize=5.0 K 2024-12-05T22:14:23,400 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6cf235ea8185447c9d66450e6eaecf62 as hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6cf235ea8185447c9d66450e6eaecf62 2024-12-05T22:14:23,406 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6cf235ea8185447c9d66450e6eaecf62, entries=1, sequenceid=104, filesize=4.9 K 2024-12-05T22:14:23,407 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.20 KB/41161, heapSize ~50.05 KB/51256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 163ms, sequenceid=104, compaction requested=false 2024-12-05T22:14:23,409 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:14:23,409 DEBUG [M:0;d029b80fd32e:35291 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:14:23,409 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/MasterData/WALs/d029b80fd32e,35291,1733436751952 2024-12-05T22:14:23,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43775 is added to blk_1073741830_1006 (size=48462) 2024-12-05T22:14:23,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44383 is added to blk_1073741830_1006 (size=48462) 2024-12-05T22:14:23,412 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:14:23,412 INFO [M:0;d029b80fd32e:35291 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-05T22:14:23,412 INFO [M:0;d029b80fd32e:35291 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:35291 2024-12-05T22:14:23,414 DEBUG [M:0;d029b80fd32e:35291 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/d029b80fd32e,35291,1733436751952 already deleted, retry=false 2024-12-05T22:14:23,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:14:23,516 INFO [M:0;d029b80fd32e:35291 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,35291,1733436751952; zookeeper connection closed. 2024-12-05T22:14:23,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35291-0x10184afa89e0000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:14:23,521 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@163cfad6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:23,523 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f952caa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:14:23,523 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:14:23,523 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6eb1b261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:14:23,523 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4debea22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.log.dir/,STOPPED} 2024-12-05T22:14:23,526 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:14:23,526 WARN [BP-716073671-172.17.0.2-1733436748740 heartbeating to localhost/127.0.0.1:37467 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:14:23,526 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:14:23,526 WARN [BP-716073671-172.17.0.2-1733436748740 heartbeating to localhost/127.0.0.1:37467 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-716073671-172.17.0.2-1733436748740 (Datanode Uuid aed8d442-5bd2-4df9-b6c0-f4129b88bed0) service to localhost/127.0.0.1:37467 2024-12-05T22:14:23,527 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/dfs/data/data3/current/BP-716073671-172.17.0.2-1733436748740 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:23,527 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/dfs/data/data4/current/BP-716073671-172.17.0.2-1733436748740 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:23,528 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:14:23,530 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6aad8790{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:23,530 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@587d1dca{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:14:23,530 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:14:23,530 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b4ce9e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:14:23,530 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2276bd44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.log.dir/,STOPPED} 2024-12-05T22:14:23,532 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:14:23,532 WARN [BP-716073671-172.17.0.2-1733436748740 heartbeating to localhost/127.0.0.1:37467 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:14:23,532 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:14:23,532 WARN [BP-716073671-172.17.0.2-1733436748740 heartbeating to localhost/127.0.0.1:37467 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-716073671-172.17.0.2-1733436748740 (Datanode Uuid 6b1ac47c-29d5-4a6f-ac3b-1dd8e06757b3) service to localhost/127.0.0.1:37467 2024-12-05T22:14:23,533 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/dfs/data/data1/current/BP-716073671-172.17.0.2-1733436748740 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:23,533 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/cluster_f27d8c50-ba9b-43d4-4b47-34ee4cc53f24/dfs/data/data2/current/BP-716073671-172.17.0.2-1733436748740 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:23,533 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:14:23,542 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5682c4d1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:14:23,543 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ff1a6c1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:14:23,543 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:14:23,543 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74468826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:14:23,543 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@88aab13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.log.dir/,STOPPED} 2024-12-05T22:14:23,553 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-05T22:14:23,588 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-05T22:14:23,596 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=63 (was 12) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37467 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37467 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37467 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37467 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/d029b80fd32e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@65f50ded java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37467 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:37467 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: RS-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/d029b80fd32e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:37467 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/d029b80fd32e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/d029b80fd32e:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37467 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37467 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=401 (was 286) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 637), ProcessCount=11 (was 11), AvailableMemoryMB=3639 (was 5284) 2024-12-05T22:14:23,602 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=64, OpenFileDescriptor=401, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=3639 2024-12-05T22:14:23,602 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.log.dir so I do NOT create it in target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/5d7da5ff-ea6a-7c50-84af-b306383243fc/hadoop.tmp.dir so I do NOT create it in target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb, deleteOnExit=true 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/test.cache.data in system properties and HBase conf 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir in system properties and HBase conf 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T22:14:23,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-05T22:14:23,603 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T22:14:23,604 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/nfs.dump.dir in system properties and HBase conf 2024-12-05T22:14:23,605 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/java.io.tmpdir in system properties and HBase conf 2024-12-05T22:14:23,605 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:14:23,605 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T22:14:23,605 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T22:14:23,618 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:14:23,696 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:14:23,702 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:14:23,703 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:14:23,703 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:14:23,703 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:14:23,703 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:14:23,704 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c5ad4dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:14:23,704 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52ca9eab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:14:23,821 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7ef935fd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/java.io.tmpdir/jetty-localhost-38007-hadoop-hdfs-3_4_1-tests_jar-_-any-3094554499106874779/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:14:23,821 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3ac35248{HTTP/1.1, (http/1.1)}{localhost:38007} 2024-12-05T22:14:23,821 INFO [Time-limited test {}] server.Server(415): Started @117539ms 2024-12-05T22:14:23,835 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:14:23,905 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:14:23,908 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:14:23,909 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:14:23,909 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:14:23,909 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:14:23,909 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a92873{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:14:23,909 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a2efd9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:14:24,031 INFO [regionserver/d029b80fd32e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:14:24,042 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44bcdc35{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/java.io.tmpdir/jetty-localhost-45801-hadoop-hdfs-3_4_1-tests_jar-_-any-13031475311311889027/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:24,042 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6282dcf3{HTTP/1.1, (http/1.1)}{localhost:45801} 2024-12-05T22:14:24,042 INFO [Time-limited test {}] server.Server(415): Started @117760ms 2024-12-05T22:14:24,044 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:14:24,079 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:14:24,082 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:14:24,083 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:14:24,083 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:14:24,083 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:14:24,084 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6fc17892{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:14:24,084 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d899dad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:14:24,158 WARN [Thread-449 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data1/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:24,158 WARN [Thread-450 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data2/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:24,188 WARN [Thread-428 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:14:24,191 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf54745a620022c8a with lease ID 0x6069641c42cadf2b: Processing first storage report for DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40 from datanode DatanodeRegistration(127.0.0.1:38977, datanodeUuid=fb9fed57-8e9e-43ca-9f66-0afbaf11957c, infoPort=38687, infoSecurePort=0, ipcPort=38899, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:24,191 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf54745a620022c8a with lease ID 0x6069641c42cadf2b: from storage DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40 node DatanodeRegistration(127.0.0.1:38977, datanodeUuid=fb9fed57-8e9e-43ca-9f66-0afbaf11957c, infoPort=38687, infoSecurePort=0, ipcPort=38899, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:24,191 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf54745a620022c8a with lease ID 0x6069641c42cadf2b: Processing first storage report for DS-b84df455-17c7-42ee-9761-bd2b41fe5662 from datanode DatanodeRegistration(127.0.0.1:38977, datanodeUuid=fb9fed57-8e9e-43ca-9f66-0afbaf11957c, infoPort=38687, infoSecurePort=0, ipcPort=38899, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:24,191 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf54745a620022c8a with lease ID 0x6069641c42cadf2b: from storage DS-b84df455-17c7-42ee-9761-bd2b41fe5662 node DatanodeRegistration(127.0.0.1:38977, datanodeUuid=fb9fed57-8e9e-43ca-9f66-0afbaf11957c, infoPort=38687, infoSecurePort=0, ipcPort=38899, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:24,211 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@76bd7720{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/java.io.tmpdir/jetty-localhost-45007-hadoop-hdfs-3_4_1-tests_jar-_-any-169548194619524097/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:24,211 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3625ff{HTTP/1.1, (http/1.1)}{localhost:45007} 2024-12-05T22:14:24,212 INFO [Time-limited test {}] server.Server(415): Started @117929ms 2024-12-05T22:14:24,214 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:14:24,323 WARN [Thread-475 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data3/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:24,324 WARN [Thread-476 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data4/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:24,347 WARN [Thread-464 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:14:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x24f16ba0c4089d40 with lease ID 0x6069641c42cadf2c: Processing first storage report for DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b from datanode DatanodeRegistration(127.0.0.1:44147, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=39033, infoSecurePort=0, ipcPort=37733, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x24f16ba0c4089d40 with lease ID 0x6069641c42cadf2c: from storage DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b node DatanodeRegistration(127.0.0.1:44147, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=39033, infoSecurePort=0, ipcPort=37733, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x24f16ba0c4089d40 with lease ID 0x6069641c42cadf2c: Processing first storage report for DS-e0828b3e-5c1c-4b6d-9c01-59b24f747ebd from datanode DatanodeRegistration(127.0.0.1:44147, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=39033, infoSecurePort=0, ipcPort=37733, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x24f16ba0c4089d40 with lease ID 0x6069641c42cadf2c: from storage DS-e0828b3e-5c1c-4b6d-9c01-59b24f747ebd node DatanodeRegistration(127.0.0.1:44147, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=39033, infoSecurePort=0, ipcPort=37733, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:24,441 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810 2024-12-05T22:14:24,444 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/zookeeper_0, clientPort=62407, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T22:14:24,445 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=62407 2024-12-05T22:14:24,445 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:24,447 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:24,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:14:24,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:14:24,458 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be with version=8 2024-12-05T22:14:24,458 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/hbase-staging 2024-12-05T22:14:24,460 INFO [Time-limited test {}] client.ConnectionUtils(129): master/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:14:24,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:14:24,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:14:24,460 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:14:24,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:14:24,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:14:24,461 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:14:24,461 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:14:24,461 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:44177 2024-12-05T22:14:24,462 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:24,463 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:24,466 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:44177 connecting to ZooKeeper ensemble=127.0.0.1:62407 2024-12-05T22:14:24,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:441770x0, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:14:24,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44177-0x10184b1624b0000 connected 2024-12-05T22:14:24,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:14:24,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:14:24,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:14:24,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44177 2024-12-05T22:14:24,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44177 2024-12-05T22:14:24,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44177 2024-12-05T22:14:24,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44177 2024-12-05T22:14:24,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44177 2024-12-05T22:14:24,491 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be, hbase.cluster.distributed=false 2024-12-05T22:14:24,509 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:14:24,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:14:24,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:14:24,509 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:14:24,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:14:24,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:14:24,509 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:14:24,509 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:14:24,510 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45395 2024-12-05T22:14:24,510 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T22:14:24,512 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T22:14:24,512 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:24,514 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:24,517 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:45395 connecting to ZooKeeper ensemble=127.0.0.1:62407 2024-12-05T22:14:24,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453950x0, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:14:24,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453950x0, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:14:24,521 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45395-0x10184b1624b0001 connected 2024-12-05T22:14:24,522 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:14:24,522 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:14:24,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45395 2024-12-05T22:14:24,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45395 2024-12-05T22:14:24,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45395 2024-12-05T22:14:24,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45395 2024-12-05T22:14:24,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45395 2024-12-05T22:14:24,530 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/d029b80fd32e,44177,1733436864460 2024-12-05T22:14:24,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:14:24,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:14:24,532 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d029b80fd32e,44177,1733436864460 2024-12-05T22:14:24,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:14:24,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:14:24,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,535 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:14:24,536 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d029b80fd32e,44177,1733436864460 from backup master directory 2024-12-05T22:14:24,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d029b80fd32e,44177,1733436864460 2024-12-05T22:14:24,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:14:24,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:14:24,539 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:14:24,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:14:24,539 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d029b80fd32e,44177,1733436864460 2024-12-05T22:14:24,549 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d029b80fd32e:44177 2024-12-05T22:14:24,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:14:24,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:14:24,554 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/hbase.id with ID: ce60f6ad-ace1-4095-8642-10dc3e297009 2024-12-05T22:14:24,567 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:24,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:14:24,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:14:24,580 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:14:24,581 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T22:14:24,581 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:14:24,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:14:24,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:14:24,590 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store 2024-12-05T22:14:24,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:14:24,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:14:24,598 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:14:24,598 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:14:24,598 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:14:24,598 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:14:24,598 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:14:24,598 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:14:24,598 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:14:24,598 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:14:24,599 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/.initializing 2024-12-05T22:14:24,599 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460 2024-12-05T22:14:24,602 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C44177%2C1733436864460, suffix=, logDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460, archiveDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/oldWALs, maxLogs=10 2024-12-05T22:14:24,602 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C44177%2C1733436864460.1733436864602 2024-12-05T22:14:24,607 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436864602 2024-12-05T22:14:24,607 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38687:38687),(127.0.0.1/127.0.0.1:39033:39033)] 2024-12-05T22:14:24,607 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:14:24,608 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:14:24,608 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:14:24,608 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:14:24,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:14:24,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T22:14:24,611 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:24,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:14:24,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:14:24,613 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T22:14:24,613 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:24,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:14:24,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:14:24,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T22:14:24,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:24,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:14:24,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:14:24,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T22:14:24,618 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:24,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:14:24,619 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:14:24,619 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:14:24,622 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T22:14:24,623 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:14:24,625 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:14:24,626 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=701756, jitterRate=-0.10767146944999695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T22:14:24,627 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:14:24,627 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T22:14:24,631 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70af6bed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:14:24,632 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-05T22:14:24,633 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T22:14:24,633 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T22:14:24,633 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T22:14:24,633 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T22:14:24,634 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-05T22:14:24,634 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T22:14:24,636 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T22:14:24,637 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T22:14:24,638 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-05T22:14:24,638 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T22:14:24,639 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T22:14:24,640 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-05T22:14:24,641 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T22:14:24,641 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T22:14:24,643 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-05T22:14:24,644 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T22:14:24,645 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T22:14:24,647 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T22:14:24,648 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T22:14:24,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:14:24,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:14:24,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,650 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=d029b80fd32e,44177,1733436864460, sessionid=0x10184b1624b0000, setting cluster-up flag (Was=false) 2024-12-05T22:14:24,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,659 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T22:14:24,660 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,44177,1733436864460 2024-12-05T22:14:24,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:24,668 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T22:14:24,670 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,44177,1733436864460 2024-12-05T22:14:24,673 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-05T22:14:24,674 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-05T22:14:24,674 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T22:14:24,674 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d029b80fd32e,44177,1733436864460 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T22:14:24,674 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:14:24,675 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:14:24,675 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:14:24,675 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:14:24,675 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d029b80fd32e:0, corePoolSize=10, maxPoolSize=10 2024-12-05T22:14:24,675 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,675 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:14:24,675 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,676 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733436894676 2024-12-05T22:14:24,677 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T22:14:24,677 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T22:14:24,677 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T22:14:24,677 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T22:14:24,677 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T22:14:24,677 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T22:14:24,677 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,677 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:14:24,678 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-05T22:14:24,678 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T22:14:24,678 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T22:14:24,678 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T22:14:24,678 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T22:14:24,678 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T22:14:24,679 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:24,679 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:14:24,680 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436864678,5,FailOnTimeoutGroup] 2024-12-05T22:14:24,681 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436864680,5,FailOnTimeoutGroup] 2024-12-05T22:14:24,681 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,681 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T22:14:24,681 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,681 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:14:24,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:14:24,689 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-05T22:14:24,689 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be 2024-12-05T22:14:24,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:14:24,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:14:24,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:14:24,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:14:24,706 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:14:24,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:24,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:14:24,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:14:24,708 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:14:24,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:24,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:14:24,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:14:24,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:14:24,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:24,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:14:24,711 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/hbase/meta/1588230740 2024-12-05T22:14:24,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/hbase/meta/1588230740 2024-12-05T22:14:24,713 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:14:24,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:14:24,717 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:14:24,717 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=722711, jitterRate=-0.08102668821811676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:14:24,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:14:24,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:14:24,718 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:14:24,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:14:24,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:14:24,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:14:24,719 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:14:24,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:14:24,720 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:14:24,720 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-05T22:14:24,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T22:14:24,722 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T22:14:24,723 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T22:14:24,742 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d029b80fd32e:45395 2024-12-05T22:14:24,744 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1008): ClusterId : ce60f6ad-ace1-4095-8642-10dc3e297009 2024-12-05T22:14:24,744 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T22:14:24,746 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T22:14:24,746 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T22:14:24,748 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T22:14:24,748 DEBUG [RS:0;d029b80fd32e:45395 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38589f03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:14:24,748 DEBUG [RS:0;d029b80fd32e:45395 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@544ba090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:14:24,749 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-05T22:14:24,749 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-05T22:14:24,749 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-05T22:14:24,749 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(3073): reportForDuty to master=d029b80fd32e,44177,1733436864460 with isa=d029b80fd32e/172.17.0.2:45395, startcode=1733436864508 2024-12-05T22:14:24,749 DEBUG [RS:0;d029b80fd32e:45395 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:14:24,752 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33553, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:14:24,752 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44177 {}] master.ServerManager(332): Checking decommissioned status of RegionServer d029b80fd32e,45395,1733436864508 2024-12-05T22:14:24,752 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44177 {}] master.ServerManager(486): Registering regionserver=d029b80fd32e,45395,1733436864508 2024-12-05T22:14:24,754 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be 2024-12-05T22:14:24,754 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:36083 2024-12-05T22:14:24,754 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-05T22:14:24,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:14:24,757 DEBUG [RS:0;d029b80fd32e:45395 {}] zookeeper.ZKUtil(111): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d029b80fd32e,45395,1733436864508 2024-12-05T22:14:24,757 WARN [RS:0;d029b80fd32e:45395 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:14:24,757 INFO [RS:0;d029b80fd32e:45395 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:14:24,758 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508 2024-12-05T22:14:24,758 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d029b80fd32e,45395,1733436864508] 2024-12-05T22:14:24,761 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-05T22:14:24,762 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T22:14:24,765 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T22:14:24,765 INFO [RS:0;d029b80fd32e:45395 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T22:14:24,765 INFO [RS:0;d029b80fd32e:45395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,765 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-05T22:14:24,766 INFO [RS:0;d029b80fd32e:45395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,766 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:14:24,767 DEBUG [RS:0;d029b80fd32e:45395 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:14:24,768 INFO [RS:0;d029b80fd32e:45395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,768 INFO [RS:0;d029b80fd32e:45395 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,768 INFO [RS:0;d029b80fd32e:45395 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,768 INFO [RS:0;d029b80fd32e:45395 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,768 INFO [RS:0;d029b80fd32e:45395 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45395,1733436864508-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:14:24,789 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T22:14:24,789 INFO [RS:0;d029b80fd32e:45395 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45395,1733436864508-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:24,804 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.Replication(204): d029b80fd32e,45395,1733436864508 started 2024-12-05T22:14:24,804 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1767): Serving as d029b80fd32e,45395,1733436864508, RpcServer on d029b80fd32e/172.17.0.2:45395, sessionid=0x10184b1624b0001 2024-12-05T22:14:24,804 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T22:14:24,804 DEBUG [RS:0;d029b80fd32e:45395 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d029b80fd32e,45395,1733436864508 2024-12-05T22:14:24,804 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,45395,1733436864508' 2024-12-05T22:14:24,804 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T22:14:24,805 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T22:14:24,805 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T22:14:24,805 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T22:14:24,805 DEBUG [RS:0;d029b80fd32e:45395 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d029b80fd32e,45395,1733436864508 2024-12-05T22:14:24,805 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,45395,1733436864508' 2024-12-05T22:14:24,806 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T22:14:24,806 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T22:14:24,806 DEBUG [RS:0;d029b80fd32e:45395 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T22:14:24,806 INFO [RS:0;d029b80fd32e:45395 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T22:14:24,806 INFO [RS:0;d029b80fd32e:45395 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T22:14:24,873 WARN [d029b80fd32e:44177 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-05T22:14:24,909 INFO [RS:0;d029b80fd32e:45395 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C45395%2C1733436864508, suffix=, logDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508, archiveDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/oldWALs, maxLogs=32 2024-12-05T22:14:24,912 INFO [RS:0;d029b80fd32e:45395 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45395%2C1733436864508.1733436864912 2024-12-05T22:14:24,919 INFO [RS:0;d029b80fd32e:45395 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 2024-12-05T22:14:24,919 DEBUG [RS:0;d029b80fd32e:45395 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38687:38687),(127.0.0.1/127.0.0.1:39033:39033)] 2024-12-05T22:14:25,123 DEBUG [d029b80fd32e:44177 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T22:14:25,124 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d029b80fd32e,45395,1733436864508 2024-12-05T22:14:25,125 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,45395,1733436864508, state=OPENING 2024-12-05T22:14:25,126 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T22:14:25,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:25,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:25,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=d029b80fd32e,45395,1733436864508}] 2024-12-05T22:14:25,129 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:14:25,129 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:14:25,281 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d029b80fd32e,45395,1733436864508 2024-12-05T22:14:25,282 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T22:14:25,284 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49590, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T22:14:25,288 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-05T22:14:25,288 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:14:25,290 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C45395%2C1733436864508.meta, suffix=.meta, logDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508, archiveDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/oldWALs, maxLogs=32 2024-12-05T22:14:25,292 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta 2024-12-05T22:14:25,298 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta 2024-12-05T22:14:25,298 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38687:38687),(127.0.0.1/127.0.0.1:39033:39033)] 2024-12-05T22:14:25,298 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:14:25,299 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T22:14:25,299 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T22:14:25,299 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T22:14:25,299 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T22:14:25,299 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:14:25,299 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-05T22:14:25,299 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-05T22:14:25,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:14:25,302 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:14:25,302 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:25,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:14:25,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:14:25,304 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:14:25,304 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:25,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:14:25,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:14:25,306 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:14:25,306 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:25,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:14:25,307 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/hbase/meta/1588230740 2024-12-05T22:14:25,308 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/hbase/meta/1588230740 2024-12-05T22:14:25,310 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:14:25,311 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:14:25,312 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=717893, jitterRate=-0.08715295791625977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:14:25,313 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:14:25,314 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733436865281 2024-12-05T22:14:25,317 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T22:14:25,317 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-05T22:14:25,317 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,45395,1733436864508 2024-12-05T22:14:25,318 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,45395,1733436864508, state=OPEN 2024-12-05T22:14:25,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:14:25,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:14:25,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:14:25,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:14:25,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T22:14:25,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=d029b80fd32e,45395,1733436864508 in 196 msec 2024-12-05T22:14:25,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T22:14:25,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec 2024-12-05T22:14:25,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 659 msec 2024-12-05T22:14:25,333 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733436865332, completionTime=-1 2024-12-05T22:14:25,333 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T22:14:25,333 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-05T22:14:25,334 DEBUG [hconnection-0x5a51fb78-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:14:25,335 INFO [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49596, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:14:25,336 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-05T22:14:25,336 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733436925336 2024-12-05T22:14:25,336 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733436985336 2024-12-05T22:14:25,336 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec 2024-12-05T22:14:25,342 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,44177,1733436864460-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,342 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,44177,1733436864460-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,342 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,44177,1733436864460-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,342 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d029b80fd32e:44177, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,342 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,342 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-05T22:14:25,342 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:14:25,344 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-05T22:14:25,344 DEBUG [master/d029b80fd32e:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-05T22:14:25,345 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:14:25,345 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:25,346 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:14:25,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:14:25,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:14:25,356 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => dd5bc4db1f09996436e67228f3bee51f, NAME => 'hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be 2024-12-05T22:14:25,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:14:25,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:14:25,364 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:14:25,365 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing dd5bc4db1f09996436e67228f3bee51f, disabling compactions & flushes 2024-12-05T22:14:25,365 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:14:25,365 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:14:25,365 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. after waiting 0 ms 2024-12-05T22:14:25,365 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:14:25,365 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:14:25,365 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for dd5bc4db1f09996436e67228f3bee51f: 2024-12-05T22:14:25,366 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:14:25,366 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733436865366"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733436865366"}]},"ts":"1733436865366"} 2024-12-05T22:14:25,369 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:14:25,370 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:14:25,370 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436865370"}]},"ts":"1733436865370"} 2024-12-05T22:14:25,372 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-05T22:14:25,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=dd5bc4db1f09996436e67228f3bee51f, ASSIGN}] 2024-12-05T22:14:25,378 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=dd5bc4db1f09996436e67228f3bee51f, ASSIGN 2024-12-05T22:14:25,379 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=dd5bc4db1f09996436e67228f3bee51f, ASSIGN; state=OFFLINE, location=d029b80fd32e,45395,1733436864508; forceNewPlan=false, retain=false 2024-12-05T22:14:25,530 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=dd5bc4db1f09996436e67228f3bee51f, regionState=OPENING, regionLocation=d029b80fd32e,45395,1733436864508 2024-12-05T22:14:25,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure dd5bc4db1f09996436e67228f3bee51f, server=d029b80fd32e,45395,1733436864508}] 2024-12-05T22:14:25,686 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d029b80fd32e,45395,1733436864508 2024-12-05T22:14:25,690 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:14:25,690 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => dd5bc4db1f09996436e67228f3bee51f, NAME => 'hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:14:25,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:14:25,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:14:25,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:14:25,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:14:25,692 INFO [StoreOpener-dd5bc4db1f09996436e67228f3bee51f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:14:25,694 INFO [StoreOpener-dd5bc4db1f09996436e67228f3bee51f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd5bc4db1f09996436e67228f3bee51f columnFamilyName info 2024-12-05T22:14:25,694 DEBUG [StoreOpener-dd5bc4db1f09996436e67228f3bee51f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:25,695 INFO [StoreOpener-dd5bc4db1f09996436e67228f3bee51f-1 {}] regionserver.HStore(327): Store=dd5bc4db1f09996436e67228f3bee51f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:14:25,696 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/hbase/namespace/dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:14:25,696 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/hbase/namespace/dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:14:25,698 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:14:25,701 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/hbase/namespace/dd5bc4db1f09996436e67228f3bee51f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:14:25,701 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened dd5bc4db1f09996436e67228f3bee51f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=695064, jitterRate=-0.1161813884973526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:14:25,702 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for dd5bc4db1f09996436e67228f3bee51f: 2024-12-05T22:14:25,703 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f., pid=6, masterSystemTime=1733436865685 2024-12-05T22:14:25,706 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:14:25,706 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:14:25,707 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=dd5bc4db1f09996436e67228f3bee51f, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,45395,1733436864508 2024-12-05T22:14:25,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T22:14:25,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure dd5bc4db1f09996436e67228f3bee51f, server=d029b80fd32e,45395,1733436864508 in 177 msec 2024-12-05T22:14:25,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T22:14:25,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=dd5bc4db1f09996436e67228f3bee51f, ASSIGN in 335 msec 2024-12-05T22:14:25,715 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:14:25,716 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436865715"}]},"ts":"1733436865715"} 2024-12-05T22:14:25,717 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-05T22:14:25,720 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:14:25,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 378 msec 2024-12-05T22:14:25,745 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-05T22:14:25,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:14:25,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:25,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:14:25,753 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-05T22:14:25,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:14:25,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 14 msec 2024-12-05T22:14:25,776 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-05T22:14:25,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:14:25,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 11 msec 2024-12-05T22:14:25,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-05T22:14:25,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-05T22:14:25,804 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.265sec 2024-12-05T22:14:25,804 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T22:14:25,804 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T22:14:25,804 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T22:14:25,804 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T22:14:25,804 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T22:14:25,804 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,44177,1733436864460-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:14:25,804 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,44177,1733436864460-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T22:14:25,806 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-05T22:14:25,806 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T22:14:25,806 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,44177,1733436864460-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,842 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31b5c2bd to 127.0.0.1:62407 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@795379a4 2024-12-05T22:14:25,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3df7da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:14:25,847 DEBUG [hconnection-0x17841abd-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:14:25,850 INFO [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49600, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:14:25,852 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=d029b80fd32e,44177,1733436864460 2024-12-05T22:14:25,853 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:25,856 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-05T22:14:25,873 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:14:25,873 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:14:25,873 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:14:25,873 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:14:25,873 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:14:25,873 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:14:25,873 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:14:25,873 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:14:25,874 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45479 2024-12-05T22:14:25,874 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T22:14:25,875 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T22:14:25,875 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:25,878 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:14:25,881 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:45479 connecting to ZooKeeper ensemble=127.0.0.1:62407 2024-12-05T22:14:25,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454790x0, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:14:25,885 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:454790x0, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:14:25,885 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45479-0x10184b1624b0003 connected 2024-12-05T22:14:25,886 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45479-0x10184b1624b0003, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-05T22:14:25,886 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45479-0x10184b1624b0003, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:14:25,887 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45479 2024-12-05T22:14:25,887 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45479 2024-12-05T22:14:25,887 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45479 2024-12-05T22:14:25,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45479 2024-12-05T22:14:25,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45479 2024-12-05T22:14:25,888 DEBUG [pool-282-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-05T22:14:25,901 DEBUG [RS:1;d029b80fd32e:45479 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d029b80fd32e:45479 2024-12-05T22:14:25,902 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1008): ClusterId : ce60f6ad-ace1-4095-8642-10dc3e297009 2024-12-05T22:14:25,902 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T22:14:25,904 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T22:14:25,904 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T22:14:25,906 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T22:14:25,906 DEBUG [RS:1;d029b80fd32e:45479 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ac239e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:14:25,907 DEBUG [RS:1;d029b80fd32e:45479 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510da607, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:14:25,907 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-05T22:14:25,907 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-05T22:14:25,907 DEBUG [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-05T22:14:25,908 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(3073): reportForDuty to master=d029b80fd32e,44177,1733436864460 with isa=d029b80fd32e/172.17.0.2:45479, startcode=1733436865872 2024-12-05T22:14:25,908 DEBUG [RS:1;d029b80fd32e:45479 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:14:25,910 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35793, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:14:25,910 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44177 {}] master.ServerManager(332): Checking decommissioned status of RegionServer d029b80fd32e,45479,1733436865872 2024-12-05T22:14:25,910 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44177 {}] master.ServerManager(486): Registering regionserver=d029b80fd32e,45479,1733436865872 2024-12-05T22:14:25,912 DEBUG [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be 2024-12-05T22:14:25,912 DEBUG [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:36083 2024-12-05T22:14:25,912 DEBUG [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-05T22:14:25,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:14:25,914 DEBUG [RS:1;d029b80fd32e:45479 {}] zookeeper.ZKUtil(111): regionserver:45479-0x10184b1624b0003, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d029b80fd32e,45479,1733436865872 2024-12-05T22:14:25,914 WARN [RS:1;d029b80fd32e:45479 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:14:25,914 INFO [RS:1;d029b80fd32e:45479 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:14:25,914 DEBUG [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45479,1733436865872 2024-12-05T22:14:25,914 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d029b80fd32e,45479,1733436865872] 2024-12-05T22:14:25,918 DEBUG [RS:1;d029b80fd32e:45479 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-05T22:14:25,918 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T22:14:25,921 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T22:14:25,921 INFO [RS:1;d029b80fd32e:45479 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T22:14:25,921 INFO [RS:1;d029b80fd32e:45479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,922 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-05T22:14:25,922 INFO [RS:1;d029b80fd32e:45479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:14:25,923 DEBUG [RS:1;d029b80fd32e:45479 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:14:25,923 INFO [RS:1;d029b80fd32e:45479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,924 INFO [RS:1;d029b80fd32e:45479 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,924 INFO [RS:1;d029b80fd32e:45479 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,924 INFO [RS:1;d029b80fd32e:45479 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,924 INFO [RS:1;d029b80fd32e:45479 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45479,1733436865872-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:14:25,940 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T22:14:25,940 INFO [RS:1;d029b80fd32e:45479 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45479,1733436865872-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:14:25,956 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.Replication(204): d029b80fd32e,45479,1733436865872 started 2024-12-05T22:14:25,956 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1767): Serving as d029b80fd32e,45479,1733436865872, RpcServer on d029b80fd32e/172.17.0.2:45479, sessionid=0x10184b1624b0003 2024-12-05T22:14:25,956 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T22:14:25,956 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3355): Started new server=Thread[RS:1;d029b80fd32e:45479,5,FailOnTimeoutGroup] 2024-12-05T22:14:25,956 DEBUG [RS:1;d029b80fd32e:45479 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d029b80fd32e,45479,1733436865872 2024-12-05T22:14:25,956 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,45479,1733436865872' 2024-12-05T22:14:25,956 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T22:14:25,956 INFO [Time-limited test {}] wal.TestLogRolling(191): Replication=2 2024-12-05T22:14:25,957 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T22:14:25,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T22:14:25,957 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T22:14:25,957 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T22:14:25,957 DEBUG [RS:1;d029b80fd32e:45479 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d029b80fd32e,45479,1733436865872 2024-12-05T22:14:25,957 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,45479,1733436865872' 2024-12-05T22:14:25,957 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T22:14:25,958 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T22:14:25,958 DEBUG [RS:1;d029b80fd32e:45479 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T22:14:25,958 INFO [RS:1;d029b80fd32e:45479 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T22:14:25,958 INFO [RS:1;d029b80fd32e:45479 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T22:14:25,960 INFO [RS-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59322, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T22:14:25,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44177 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-05T22:14:25,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44177 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-05T22:14:25,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44177 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:14:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44177 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-12-05T22:14:25,966 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:14:25,966 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:25,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44177 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 9 2024-12-05T22:14:25,967 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:14:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44177 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:14:25,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741837_1013 (size=393) 2024-12-05T22:14:25,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741837_1013 (size=393) 2024-12-05T22:14:25,979 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 34c26a837ba6f5545f8521b057bbf435, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be 2024-12-05T22:14:25,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44147 is added to blk_1073741838_1014 (size=76) 2024-12-05T22:14:25,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38977 is added to blk_1073741838_1014 (size=76) 2024-12-05T22:14:25,987 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:14:25,987 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1681): Closing 34c26a837ba6f5545f8521b057bbf435, disabling compactions & flushes 2024-12-05T22:14:25,987 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:14:25,987 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:14:25,987 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. after waiting 0 ms 2024-12-05T22:14:25,987 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:14:25,987 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:14:25,987 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1635): Region close journal for 34c26a837ba6f5545f8521b057bbf435: 2024-12-05T22:14:25,989 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:14:25,989 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733436865989"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733436865989"}]},"ts":"1733436865989"} 2024-12-05T22:14:25,991 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:14:25,992 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:14:25,993 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436865992"}]},"ts":"1733436865992"} 2024-12-05T22:14:25,994 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-12-05T22:14:25,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=34c26a837ba6f5545f8521b057bbf435, ASSIGN}] 2024-12-05T22:14:26,000 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=34c26a837ba6f5545f8521b057bbf435, ASSIGN 2024-12-05T22:14:26,001 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=34c26a837ba6f5545f8521b057bbf435, ASSIGN; state=OFFLINE, location=d029b80fd32e,45395,1733436864508; forceNewPlan=false, retain=false 2024-12-05T22:14:26,061 INFO [RS:1;d029b80fd32e:45479 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C45479%2C1733436865872, suffix=, logDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45479,1733436865872, archiveDir=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/oldWALs, maxLogs=32 2024-12-05T22:14:26,062 INFO [RS:1;d029b80fd32e:45479 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45479%2C1733436865872.1733436866062 2024-12-05T22:14:26,069 INFO [RS:1;d029b80fd32e:45479 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45479,1733436865872/d029b80fd32e%2C45479%2C1733436865872.1733436866062 2024-12-05T22:14:26,070 DEBUG [RS:1;d029b80fd32e:45479 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38687:38687),(127.0.0.1/127.0.0.1:39033:39033)] 2024-12-05T22:14:26,153 INFO [d029b80fd32e:44177 {}] balancer.BaseLoadBalancer(546): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T22:14:26,153 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=34c26a837ba6f5545f8521b057bbf435, regionState=OPENING, regionLocation=d029b80fd32e,45395,1733436864508 2024-12-05T22:14:26,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 34c26a837ba6f5545f8521b057bbf435, server=d029b80fd32e,45395,1733436864508}] 2024-12-05T22:14:26,309 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d029b80fd32e,45395,1733436864508 2024-12-05T22:14:26,313 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:14:26,313 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 34c26a837ba6f5545f8521b057bbf435, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:14:26,314 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:14:26,314 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:14:26,314 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:14:26,314 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:14:26,316 INFO [StoreOpener-34c26a837ba6f5545f8521b057bbf435-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:14:26,318 INFO [StoreOpener-34c26a837ba6f5545f8521b057bbf435-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34c26a837ba6f5545f8521b057bbf435 columnFamilyName info 2024-12-05T22:14:26,318 DEBUG [StoreOpener-34c26a837ba6f5545f8521b057bbf435-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:14:26,318 INFO [StoreOpener-34c26a837ba6f5545f8521b057bbf435-1 {}] regionserver.HStore(327): Store=34c26a837ba6f5545f8521b057bbf435/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:14:26,319 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:14:26,319 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:14:26,322 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:14:26,324 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:14:26,324 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 34c26a837ba6f5545f8521b057bbf435; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=739861, jitterRate=-0.05921828746795654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:14:26,325 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 34c26a837ba6f5545f8521b057bbf435: 2024-12-05T22:14:26,326 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435., pid=11, masterSystemTime=1733436866309 2024-12-05T22:14:26,329 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:14:26,329 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:14:26,329 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=34c26a837ba6f5545f8521b057bbf435, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,45395,1733436864508 2024-12-05T22:14:26,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-05T22:14:26,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 34c26a837ba6f5545f8521b057bbf435, server=d029b80fd32e,45395,1733436864508 in 175 msec 2024-12-05T22:14:26,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T22:14:26,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=34c26a837ba6f5545f8521b057bbf435, ASSIGN in 335 msec 2024-12-05T22:14:26,338 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:14:26,338 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436866338"}]},"ts":"1733436866338"} 2024-12-05T22:14:26,340 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-12-05T22:14:26,343 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:14:26,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 381 msec 2024-12-05T22:14:28,085 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:14:28,098 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:14:28,613 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:14:28,614 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:14:28,649 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:14:30,762 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T22:14:30,763 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-05T22:14:30,764 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-12-05T22:14:32,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-05T22:14:32,541 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-12-05T22:14:32,543 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-05T22:14:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44177 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:14:35,970 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath, procId: 9 completed 2024-12-05T22:14:35,973 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-12-05T22:14:35,974 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:14:35,995 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:14:35,999 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:14:36,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:14:36,000 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:14:36,001 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:14:36,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@325c79fd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:14:36,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d729c6d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:14:36,167 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1ad056f1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/java.io.tmpdir/jetty-localhost-42459-hadoop-hdfs-3_4_1-tests_jar-_-any-16734620195321587725/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:36,171 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@393cbee7{HTTP/1.1, (http/1.1)}{localhost:42459} 2024-12-05T22:14:36,171 INFO [Time-limited test {}] server.Server(415): Started @129888ms 2024-12-05T22:14:36,173 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:14:36,228 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:14:36,233 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:14:36,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:14:36,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:14:36,235 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:14:36,235 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29567200{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:14:36,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6944f542{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:14:36,305 WARN [Thread-631 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data5/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:36,305 WARN [Thread-632 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data6/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:36,333 WARN [Thread-611 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:14:36,338 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96a278b14bd614a1 with lease ID 0x6069641c42cadf2d: Processing first storage report for DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d from datanode DatanodeRegistration(127.0.0.1:45699, datanodeUuid=470227bd-9ec8-4839-a4b3-58f7f25ba0f2, infoPort=39917, infoSecurePort=0, ipcPort=37703, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:36,338 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96a278b14bd614a1 with lease ID 0x6069641c42cadf2d: from storage DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d node DatanodeRegistration(127.0.0.1:45699, datanodeUuid=470227bd-9ec8-4839-a4b3-58f7f25ba0f2, infoPort=39917, infoSecurePort=0, ipcPort=37703, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:36,338 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96a278b14bd614a1 with lease ID 0x6069641c42cadf2d: Processing first storage report for DS-64ff4363-a766-48eb-94f7-30ca5249873d from datanode DatanodeRegistration(127.0.0.1:45699, datanodeUuid=470227bd-9ec8-4839-a4b3-58f7f25ba0f2, infoPort=39917, infoSecurePort=0, ipcPort=37703, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:36,338 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96a278b14bd614a1 with lease ID 0x6069641c42cadf2d: from storage DS-64ff4363-a766-48eb-94f7-30ca5249873d node DatanodeRegistration(127.0.0.1:45699, datanodeUuid=470227bd-9ec8-4839-a4b3-58f7f25ba0f2, infoPort=39917, infoSecurePort=0, ipcPort=37703, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:36,369 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4e27bb93{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/java.io.tmpdir/jetty-localhost-33835-hadoop-hdfs-3_4_1-tests_jar-_-any-13564581945090227883/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:36,369 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@693baa47{HTTP/1.1, (http/1.1)}{localhost:33835} 2024-12-05T22:14:36,369 INFO [Time-limited test {}] server.Server(415): Started @130087ms 2024-12-05T22:14:36,371 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:14:36,409 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:14:36,413 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:14:36,416 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:14:36,416 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:14:36,416 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:14:36,416 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40616806{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:14:36,417 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@612cf16{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:14:36,523 WARN [Thread-666 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data7/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:36,523 WARN [Thread-667 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data8/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:36,536 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@560986a2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/java.io.tmpdir/jetty-localhost-38113-hadoop-hdfs-3_4_1-tests_jar-_-any-5102341192985483757/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:36,537 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1e28d6de{HTTP/1.1, (http/1.1)}{localhost:38113} 2024-12-05T22:14:36,537 INFO [Time-limited test {}] server.Server(415): Started @130254ms 2024-12-05T22:14:36,538 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:14:36,543 WARN [Thread-646 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:14:36,546 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe6ab7848d478ccf1 with lease ID 0x6069641c42cadf2e: Processing first storage report for DS-025f710e-5f95-4709-9682-19cd7c707102 from datanode DatanodeRegistration(127.0.0.1:45671, datanodeUuid=307bcc59-80b2-4552-ba8b-f0f24a0104a9, infoPort=39523, infoSecurePort=0, ipcPort=33537, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:36,546 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6ab7848d478ccf1 with lease ID 0x6069641c42cadf2e: from storage DS-025f710e-5f95-4709-9682-19cd7c707102 node DatanodeRegistration(127.0.0.1:45671, datanodeUuid=307bcc59-80b2-4552-ba8b-f0f24a0104a9, infoPort=39523, infoSecurePort=0, ipcPort=33537, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:36,546 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe6ab7848d478ccf1 with lease ID 0x6069641c42cadf2e: Processing first storage report for DS-64559f70-b3ec-460d-aecc-3b0b2e44d48d from datanode DatanodeRegistration(127.0.0.1:45671, datanodeUuid=307bcc59-80b2-4552-ba8b-f0f24a0104a9, infoPort=39523, infoSecurePort=0, ipcPort=33537, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:36,547 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6ab7848d478ccf1 with lease ID 0x6069641c42cadf2e: from storage DS-64559f70-b3ec-460d-aecc-3b0b2e44d48d node DatanodeRegistration(127.0.0.1:45671, datanodeUuid=307bcc59-80b2-4552-ba8b-f0f24a0104a9, infoPort=39523, infoSecurePort=0, ipcPort=33537, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:36,776 WARN [Thread-693 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data10/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:36,776 WARN [Thread-692 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data9/current/BP-1883352144-172.17.0.2-1733436863637/current, will proceed with Du for space computation calculation, 2024-12-05T22:14:36,795 WARN [Thread-681 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:14:36,798 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x323785289b263198 with lease ID 0x6069641c42cadf2f: Processing first storage report for DS-5540c73b-b976-4044-aa80-78188185063b from datanode DatanodeRegistration(127.0.0.1:42417, datanodeUuid=ec5d95b1-3abc-49a7-8f6e-2970b8c20258, infoPort=42965, infoSecurePort=0, ipcPort=38777, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:36,798 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x323785289b263198 with lease ID 0x6069641c42cadf2f: from storage DS-5540c73b-b976-4044-aa80-78188185063b node DatanodeRegistration(127.0.0.1:42417, datanodeUuid=ec5d95b1-3abc-49a7-8f6e-2970b8c20258, infoPort=42965, infoSecurePort=0, ipcPort=38777, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:36,798 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x323785289b263198 with lease ID 0x6069641c42cadf2f: Processing first storage report for DS-901015c1-1d5a-448c-b4ec-7570804830e0 from datanode DatanodeRegistration(127.0.0.1:42417, datanodeUuid=ec5d95b1-3abc-49a7-8f6e-2970b8c20258, infoPort=42965, infoSecurePort=0, ipcPort=38777, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637) 2024-12-05T22:14:36,798 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x323785289b263198 with lease ID 0x6069641c42cadf2f: from storage DS-901015c1-1d5a-448c-b4ec-7570804830e0 node DatanodeRegistration(127.0.0.1:42417, datanodeUuid=ec5d95b1-3abc-49a7-8f6e-2970b8c20258, infoPort=42965, infoSecurePort=0, ipcPort=38777, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:36,862 WARN [ResponseProcessor for block BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,862 WARN [ResponseProcessor for block BP-1883352144-172.17.0.2-1733436863637:blk_1073741839_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1883352144-172.17.0.2-1733436863637:blk_1073741839_1015 java.io.IOException: Bad response ERROR for BP-1883352144-172.17.0.2-1733436863637:blk_1073741839_1015 from datanode DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,862 WARN [ResponseProcessor for block BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,868 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 block BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK], DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]) is bad. 2024-12-05T22:14:36,868 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta block BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK], DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]) is bad. 2024-12-05T22:14:36,869 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45479,1733436865872/d029b80fd32e%2C45479%2C1733436865872.1733436866062 block BP-1883352144-172.17.0.2-1733436863637:blk_1073741839_1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741839_1015 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK], DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]) is bad. 2024-12-05T22:14:36,868 WARN [PacketResponder: BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44147] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,868 WARN [PacketResponder: BP-1883352144-172.17.0.2-1733436863637:blk_1073741839_1015, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44147] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,870 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@76bd7720{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:36,862 WARN [ResponseProcessor for block BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,870 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436864602 block BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK], DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]) is bad. 2024-12-05T22:14:36,870 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3625ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:14:36,870 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:14:36,870 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1264086765_22 at /127.0.0.1:55142 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:38977:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55142 dst: /127.0.0.1:38977 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,871 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d899dad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:14:36,871 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6fc17892{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,STOPPED} 2024-12-05T22:14:36,871 WARN [PacketResponder: BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44147] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,871 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1264086765_22 at /127.0.0.1:55500 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:44147:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55500 dst: /127.0.0.1:44147 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,871 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55084 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:38977:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55084 dst: /127.0.0.1:38977 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,874 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:14:36,874 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:14:36,874 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1883352144-172.17.0.2-1733436863637 (Datanode Uuid 730af7b4-0272-418a-88c0-5cf7aea64c4c) service to localhost/127.0.0.1:36083 2024-12-05T22:14:36,873 WARN [PacketResponder: BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44147] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,874 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:14:36,873 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2143815349_22 at /127.0.0.1:55402 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44147:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55402 dst: /127.0.0.1:44147 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,872 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2143815349_22 at /127.0.0.1:55034 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:38977:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55034 dst: /127.0.0.1:38977 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,875 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data3/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:36,873 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55432 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44147:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55432 dst: /127.0.0.1:44147 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55442 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44147:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55442 dst: /127.0.0.1:44147 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 49466 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,875 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55100 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38977:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55100 dst: /127.0.0.1:38977 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,878 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:14:36,878 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data4/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:36,878 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta block BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,878 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436864602 block BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,880 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45479,1733436865872/d029b80fd32e%2C45479%2C1733436865872.1733436866062 block BP-1883352144-172.17.0.2-1733436863637:blk_1073741839_1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1015 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,882 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@3aa0346a {}] datanode.DataXceiver(331): 127.0.0.1:38977:DataXceiver error processing unknown operation src: /127.0.0.1:54648 dst: /127.0.0.1:38977 java.io.IOException: Server closed. at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,882 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2143815349_22 at /127.0.0.1:54626 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:38977:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54626 dst: /127.0.0.1:38977 java.io.IOException: The stream is closed at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:914) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,882 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:54632 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38977:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54632 dst: /127.0.0.1:38977 java.io.IOException: The stream is closed at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:914) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:36,882 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 block BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,887 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44bcdc35{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:36,887 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6282dcf3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:14:36,887 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:14:36,887 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a2efd9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:14:36,887 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a92873{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,STOPPED} 2024-12-05T22:14:36,889 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:14:36,889 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:14:36,889 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1883352144-172.17.0.2-1733436863637 (Datanode Uuid fb9fed57-8e9e-43ca-9f66-0afbaf11957c) service to localhost/127.0.0.1:36083 2024-12-05T22:14:36,889 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:14:36,889 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data1/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:36,890 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data2/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:36,890 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:14:36,894 WARN [RS:0;d029b80fd32e:45395.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=4, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,894 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C45395%2C1733436864508:(num 1733436864912) roll requested 2024-12-05T22:14:36,894 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45395%2C1733436864508.1733436876894 2024-12-05T22:14:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:49600 deadline: 1733436886893, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL 2024-12-05T22:14:36,903 WARN [regionserver/d029b80fd32e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL 2024-12-05T22:14:36,903 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 with entries=4, filesize=959 B; new WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436876894 2024-12-05T22:14:36,904 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39523:39523),(127.0.0.1/127.0.0.1:39917:39917)] 2024-12-05T22:14:36,904 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 is not closed yet, will try archiving it next time 2024-12-05T22:14:36,904 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,904 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:36,905 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-05T22:14:36,905 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-05T22:14:36,905 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 2024-12-05T22:14:36,909 WARN [IPC Server handler 2 on default port 36083 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 has not been closed. Lease recovery is in progress. RecoveryId = 1021 for block blk_1073741833_1009 2024-12-05T22:14:36,911 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 after 6ms 2024-12-05T22:14:38,045 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:14:38,047 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:14:38,070 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:14:38,073 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:14:38,074 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:14:40,912 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 after 4007ms 2024-12-05T22:14:48,933 INFO [Time-limited test {}] wal.TestLogRolling(243): log.getCurrentFileName(): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436876894 2024-12-05T22:14:48,933 WARN [ResponseProcessor for block BP-1883352144-172.17.0.2-1733436863637:blk_1073741840_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1883352144-172.17.0.2-1733436863637:blk_1073741840_1020 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:48,934 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436876894 block BP-1883352144-172.17.0.2-1733436863637:blk_1073741840_1020 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741840_1020 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK], DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK]) is bad. 2024-12-05T22:14:48,934 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:45444 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:45671:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45444 dst: /127.0.0.1:45671 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:48,935 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:45860 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:45699:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45860 dst: /127.0.0.1:45699 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:48,936 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4e27bb93{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:48,937 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@693baa47{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:14:48,937 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:14:48,937 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6944f542{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:14:48,937 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29567200{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,STOPPED} 2024-12-05T22:14:48,940 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:14:48,940 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:14:48,940 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1883352144-172.17.0.2-1733436863637 (Datanode Uuid 307bcc59-80b2-4552-ba8b-f0f24a0104a9) service to localhost/127.0.0.1:36083 2024-12-05T22:14:48,940 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:14:48,940 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data7/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:48,940 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data8/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:48,941 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:14:48,943 WARN [sync.1 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]] 2024-12-05T22:14:48,944 WARN [sync.1 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]] 2024-12-05T22:14:48,944 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C45395%2C1733436864508:(num 1733436876894) roll requested 2024-12-05T22:14:48,944 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45395%2C1733436864508.1733436888944 2024-12-05T22:14:48,948 WARN [Thread-716 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44147 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:48,948 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:57330 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data10]'}, localName='127.0.0.1:42417', datanodeUuid='ec5d95b1-3abc-49a7-8f6e-2970b8c20258', xmitsInProgress=0}:Exception transferring block BP-1883352144-172.17.0.2-1733436863637:blk_1073741841_1023 to mirror 127.0.0.1:44147 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:48,949 WARN [Thread-716 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK], DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]) is bad. 2024-12-05T22:14:48,949 WARN [Thread-716 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741841_1023 2024-12-05T22:14:48,949 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:57330 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-05T22:14:48,949 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:57330 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:42417:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57330 dst: /127.0.0.1:42417 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:48,952 WARN [Thread-716 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK] 2024-12-05T22:14:48,956 WARN [Thread-716 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:48,956 WARN [Thread-716 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK], DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]) is bad. 2024-12-05T22:14:48,956 WARN [Thread-716 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741842_1024 2024-12-05T22:14:48,957 WARN [Thread-716 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK] 2024-12-05T22:14:48,958 WARN [Thread-716 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1025 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:48,959 WARN [Thread-716 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK]) is bad. 2024-12-05T22:14:48,959 WARN [Thread-716 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741843_1025 2024-12-05T22:14:48,959 WARN [Thread-716 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK] 2024-12-05T22:14:48,971 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436876894 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436888944 2024-12-05T22:14:48,972 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39917:39917),(127.0.0.1/127.0.0.1:42965:42965)] 2024-12-05T22:14:48,972 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 is not closed yet, will try archiving it next time 2024-12-05T22:14:48,972 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436876894 is not closed yet, will try archiving it next time 2024-12-05T22:14:48,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45699 is added to blk_1073741840_1022 (size=2431) 2024-12-05T22:14:49,377 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 is not closed yet, will try archiving it next time 2024-12-05T22:14:51,350 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6c27c657[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45699, datanodeUuid=470227bd-9ec8-4839-a4b3-58f7f25ba0f2, infoPort=39917, infoSecurePort=0, ipcPort=37703, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741840_1022 to 127.0.0.1:44147 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:52,947 WARN [ResponseProcessor for block BP-1883352144-172.17.0.2-1733436863637:blk_1073741844_1026 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1883352144-172.17.0.2-1733436863637:blk_1073741844_1026 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,948 WARN [DataStreamer for file /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436888944 block BP-1883352144-172.17.0.2-1733436863637:blk_1073741844_1026 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741844_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]) is bad. 2024-12-05T22:14:52,948 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:60368 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:45699:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60368 dst: /127.0.0.1:45699 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:52,948 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:57336 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:42417:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57336 dst: /127.0.0.1:42417 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:52,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1ad056f1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:52,951 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@393cbee7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:14:52,951 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:14:52,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d729c6d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:14:52,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@325c79fd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,STOPPED} 2024-12-05T22:14:52,953 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:14:52,953 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:14:52,954 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1883352144-172.17.0.2-1733436863637 (Datanode Uuid 470227bd-9ec8-4839-a4b3-58f7f25ba0f2) service to localhost/127.0.0.1:36083 2024-12-05T22:14:52,954 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:14:52,954 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data5/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:52,955 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data6/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:14:52,955 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:14:52,957 WARN [sync.4 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]] 2024-12-05T22:14:52,957 WARN [sync.4 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]] 2024-12-05T22:14:52,957 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C45395%2C1733436864508:(num 1733436888944) roll requested 2024-12-05T22:14:52,958 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45395%2C1733436864508.1733436892958 2024-12-05T22:14:52,961 WARN [Thread-728 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,961 WARN [Thread-728 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK], DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]) is bad. 2024-12-05T22:14:52,961 WARN [Thread-728 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741845_1028 2024-12-05T22:14:52,962 WARN [Thread-728 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK] 2024-12-05T22:14:52,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] regionserver.HRegion(8581): Flush requested on 34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:14:52,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 34c26a837ba6f5545f8521b057bbf435 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:14:52,964 WARN [Thread-728 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,964 WARN [Thread-728 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK], DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]) is bad. 2024-12-05T22:14:52,964 WARN [Thread-728 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741846_1029 2024-12-05T22:14:52,964 WARN [Thread-728 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK] 2024-12-05T22:14:52,966 WARN [Thread-728 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,966 WARN [Thread-728 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK]) is bad. 2024-12-05T22:14:52,966 WARN [Thread-728 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741847_1030 2024-12-05T22:14:52,967 WARN [Thread-728 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK] 2024-12-05T22:14:52,969 WARN [Thread-728 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,969 WARN [Thread-728 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]) is bad. 2024-12-05T22:14:52,969 WARN [Thread-728 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741848_1031 2024-12-05T22:14:52,970 WARN [Thread-728 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK] 2024-12-05T22:14:52,970 WARN [IPC Server handler 0 on default port 36083 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-05T22:14:52,971 WARN [IPC Server handler 0 on default port 36083 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-05T22:14:52,971 WARN [IPC Server handler 0 on default port 36083 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-05T22:14:52,981 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436888944 with entries=9, filesize=9.53 KB; new WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436892958 2024-12-05T22:14:52,981 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965)] 2024-12-05T22:14:52,981 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 is not closed yet, will try archiving it next time 2024-12-05T22:14:52,981 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436888944 is not closed yet, will try archiving it next time 2024-12-05T22:14:52,982 WARN [sync.3 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]] 2024-12-05T22:14:52,982 WARN [sync.3 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]] 2024-12-05T22:14:52,983 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C45395%2C1733436864508:(num 1733436892958) roll requested 2024-12-05T22:14:52,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741844_1027 (size=9768) 2024-12-05T22:14:52,983 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45395%2C1733436864508.1733436892983 2024-12-05T22:14:52,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/.tmp/info/27f01e6124d64d10bd87e10401bcbd32 is 1080, key is row0002/info:/1733436888942/Put/seqid=0 2024-12-05T22:14:52,989 WARN [Thread-733 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,989 WARN [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,989 WARN [Thread-733 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]) is bad. 2024-12-05T22:14:52,989 WARN [Thread-733 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741851_1034 2024-12-05T22:14:52,989 WARN [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]) is bad. 2024-12-05T22:14:52,989 WARN [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741850_1033 2024-12-05T22:14:52,990 WARN [Thread-733 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK] 2024-12-05T22:14:52,990 WARN [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK] 2024-12-05T22:14:52,991 WARN [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,991 WARN [Thread-733 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,991 WARN [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK], DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK]) is bad. 2024-12-05T22:14:52,991 WARN [Thread-733 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK], DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]) is bad. 2024-12-05T22:14:52,991 WARN [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741853_1036 2024-12-05T22:14:52,991 WARN [Thread-733 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741852_1035 2024-12-05T22:14:52,992 WARN [Thread-733 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK] 2024-12-05T22:14:52,992 WARN [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK] 2024-12-05T22:14:52,993 WARN [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,993 WARN [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK], DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]) is bad. 2024-12-05T22:14:52,993 WARN [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741855_1038 2024-12-05T22:14:52,994 WARN [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK] 2024-12-05T22:14:52,994 WARN [Thread-733 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38977 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,994 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:54702 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data10]'}, localName='127.0.0.1:42417', datanodeUuid='ec5d95b1-3abc-49a7-8f6e-2970b8c20258', xmitsInProgress=0}:Exception transferring block BP-1883352144-172.17.0.2-1733436863637:blk_1073741854_1037 to mirror 127.0.0.1:38977 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:52,994 WARN [Thread-733 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK], DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]) is bad. 2024-12-05T22:14:52,994 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:54702 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-05T22:14:52,994 WARN [Thread-733 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741854_1037 2024-12-05T22:14:52,994 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:54702 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:42417:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54702 dst: /127.0.0.1:42417 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:52,995 WARN [Thread-733 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK] 2024-12-05T22:14:52,995 WARN [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:52,995 WARN [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK]) is bad. 2024-12-05T22:14:52,995 WARN [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741856_1039 2024-12-05T22:14:52,996 WARN [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44147,DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b,DISK] 2024-12-05T22:14:52,996 WARN [IPC Server handler 0 on default port 36083 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-05T22:14:52,996 WARN [IPC Server handler 0 on default port 36083 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-05T22:14:52,996 WARN [IPC Server handler 0 on default port 36083 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-05T22:14:52,999 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:54714 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data10]'}, localName='127.0.0.1:42417', datanodeUuid='ec5d95b1-3abc-49a7-8f6e-2970b8c20258', xmitsInProgress=0}:Exception transferring block BP-1883352144-172.17.0.2-1733436863637:blk_1073741857_1040 to mirror 127.0.0.1:45671 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:53,000 WARN [Thread-733 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45671 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:53,000 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:54714 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-05T22:14:53,000 WARN [Thread-733 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK], DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK]) is bad. 2024-12-05T22:14:53,000 WARN [Thread-733 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741857_1040 2024-12-05T22:14:53,000 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:54714 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:42417:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54714 dst: /127.0.0.1:42417 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:53,001 WARN [Thread-733 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK] 2024-12-05T22:14:53,001 WARN [IPC Server handler 2 on default port 36083 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-05T22:14:53,001 WARN [IPC Server handler 2 on default port 36083 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-05T22:14:53,001 WARN [IPC Server handler 2 on default port 36083 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-05T22:14:53,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741858_1041 (size=10347) 2024-12-05T22:14:53,009 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436892958 with entries=4, filesize=4.65 KB; new WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436892983 2024-12-05T22:14:53,009 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965)] 2024-12-05T22:14:53,009 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 is not closed yet, will try archiving it next time 2024-12-05T22:14:53,009 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436888944 is not closed yet, will try archiving it next time 2024-12-05T22:14:53,009 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436892958 is not closed yet, will try archiving it next time 2024-12-05T22:14:53,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741849_1032 (size=4768) 2024-12-05T22:14:53,190 WARN [sync.3 {}] wal.FSHLog(760): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-12-05T22:14:53,385 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 is not closed yet, will try archiving it next time 2024-12-05T22:14:53,385 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436892958 is not closed yet, will try archiving it next time 2024-12-05T22:14:53,401 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:14:53,404 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:14:53,405 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:14:53,405 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:14:53,405 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:14:53,406 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d021301{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:14:53,406 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3295b233{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:14:53,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/.tmp/info/27f01e6124d64d10bd87e10401bcbd32 2024-12-05T22:14:53,412 DEBUG [Close-WAL-Writer-2 {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436864912 is not closed yet, will try archiving it next time 2024-12-05T22:14:53,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/.tmp/info/27f01e6124d64d10bd87e10401bcbd32 as hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/info/27f01e6124d64d10bd87e10401bcbd32 2024-12-05T22:14:53,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/info/27f01e6124d64d10bd87e10401bcbd32, entries=5, sequenceid=12, filesize=10.1 K 2024-12-05T22:14:53,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=8.40 KB/8606 for 34c26a837ba6f5545f8521b057bbf435 in 461ms, sequenceid=12, compaction requested=false 2024-12-05T22:14:53,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 34c26a837ba6f5545f8521b057bbf435: 2024-12-05T22:14:53,555 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2aec2b59{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/java.io.tmpdir/jetty-localhost-43873-hadoop-hdfs-3_4_1-tests_jar-_-any-15781366892532547415/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:14:53,555 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a79e9ef{HTTP/1.1, (http/1.1)}{localhost:43873} 2024-12-05T22:14:53,556 INFO [Time-limited test {}] server.Server(415): Started @147273ms 2024-12-05T22:14:53,557 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:14:53,668 WARN [Thread-757 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:14:53,679 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3ef9e94b3cf22f89 with lease ID 0x6069641c42cadf30: from storage DS-efe4522d-c144-43c5-b93c-bd67aedf1e8b node DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:53,680 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3ef9e94b3cf22f89 with lease ID 0x6069641c42cadf30: from storage DS-e0828b3e-5c1c-4b6d-9c01-59b24f747ebd node DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:14:54,441 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:14:54,677 WARN [master/d029b80fd32e:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=96, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:54,678 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C44177%2C1733436864460:(num 1733436864602) roll requested 2024-12-05T22:14:54,678 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:54,679 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C44177%2C1733436864460.1733436894678 2024-12-05T22:14:54,678 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:54,682 WARN [Thread-777 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:54,683 WARN [Thread-777 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK]) is bad. 2024-12-05T22:14:54,683 WARN [Thread-777 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741860_1043 2024-12-05T22:14:54,684 WARN [Thread-777 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45671,DS-025f710e-5f95-4709-9682-19cd7c707102,DISK] 2024-12-05T22:14:54,685 WARN [Thread-777 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:54,685 WARN [Thread-777 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]) is bad. 2024-12-05T22:14:54,685 WARN [Thread-777 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741861_1044 2024-12-05T22:14:54,686 WARN [Thread-777 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK] 2024-12-05T22:14:54,687 WARN [Thread-777 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:54,688 WARN [Thread-777 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK], DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]) is bad. 2024-12-05T22:14:54,688 WARN [Thread-777 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741862_1045 2024-12-05T22:14:54,688 WARN [Thread-777 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK] 2024-12-05T22:14:54,693 WARN [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL 2024-12-05T22:14:54,693 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436864602 with entries=93, filesize=46.04 KB; new WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436894678 2024-12-05T22:14:54,694 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38661:38661),(127.0.0.1/127.0.0.1:42965:42965)] 2024-12-05T22:14:54,694 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436864602 is not closed yet, will try archiving it next time 2024-12-05T22:14:54,694 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:54,694 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:14:54,694 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436864602 2024-12-05T22:14:54,694 WARN [IPC Server handler 0 on default port 36083 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436864602 has not been closed. Lease recovery is in progress. RecoveryId = 1047 for block blk_1073741830_1006 2024-12-05T22:14:54,695 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436864602 after 1ms 2024-12-05T22:14:54,799 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@ed2f66e[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42417, datanodeUuid=ec5d95b1-3abc-49a7-8f6e-2970b8c20258, infoPort=42965, infoSecurePort=0, ipcPort=38777, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741858_1041 to 127.0.0.1:45699 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:54,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741844_1027 (size=9768) 2024-12-05T22:14:55,799 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@20fd9877[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42417, datanodeUuid=ec5d95b1-3abc-49a7-8f6e-2970b8c20258, infoPort=42965, infoSecurePort=0, ipcPort=38777, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741849_1032 to 127.0.0.1:45699 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:14:58,696 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460/d029b80fd32e%2C44177%2C1733436864460.1733436864602 after 4002ms 2024-12-05T22:14:58,748 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:14:58,750 INFO [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36404, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:14:59,907 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:14:59,909 INFO [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36412, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:15:03,696 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@69acd01f {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:38977,null,null]) java.net.ConnectException: Call From d029b80fd32e/172.17.0.2 to localhost:38899 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-05T22:15:03,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741833_1021 (size=959) 2024-12-05T22:15:03,936 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436876894 to hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/oldWALs/d029b80fd32e%2C45395%2C1733436864508.1733436876894 2024-12-05T22:15:06,678 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@580ffa77[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741833_1021 to 127.0.0.1:45671 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:06,679 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6203ba3[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741837_1013 to 127.0.0.1:45671 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:06,759 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T22:15:06,759 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T22:15:07,677 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6203ba3[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741835_1011 to 127.0.0.1:45699 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:07,677 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@580ffa77[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741831_1007 to 127.0.0.1:45699 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:09,677 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@580ffa77[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741836_1012 to 127.0.0.1:45699 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:09,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:15:10,677 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@580ffa77[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741826_1002 to 127.0.0.1:45699 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:10,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:15:11,314 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 34c26a837ba6f5545f8521b057bbf435, had cached 0 bytes from a total of 10347 2024-12-05T22:15:12,350 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45395%2C1733436864508.1733436912350 2024-12-05T22:15:12,354 WARN [Thread-794 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1048 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45699 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:12,354 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2143815349_22 at /127.0.0.1:55426 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741864_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data10]'}, localName='127.0.0.1:42417', datanodeUuid='ec5d95b1-3abc-49a7-8f6e-2970b8c20258', xmitsInProgress=0}:Exception transferring block BP-1883352144-172.17.0.2-1733436863637:blk_1073741864_1048 to mirror 127.0.0.1:45699 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:12,355 WARN [Thread-794 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741864_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK], DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]) is bad. 2024-12-05T22:15:12,355 WARN [Thread-794 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741864_1048 2024-12-05T22:15:12,355 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2143815349_22 at /127.0.0.1:55426 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741864_1048] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-05T22:15:12,355 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2143815349_22 at /127.0.0.1:55426 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741864_1048] {}] datanode.DataXceiver(331): 127.0.0.1:42417:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55426 dst: /127.0.0.1:42417 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:12,355 WARN [Thread-794 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK] 2024-12-05T22:15:12,361 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436892983 with entries=2, filesize=1.57 KB; new WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436912350 2024-12-05T22:15:12,361 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38661:38661)] 2024-12-05T22:15:12,361 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436892983 is not closed yet, will try archiving it next time 2024-12-05T22:15:12,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741859_1042 (size=1618) 2024-12-05T22:15:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45395 {}] regionserver.HRegion(8581): Flush requested on 34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:15:12,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 34c26a837ba6f5545f8521b057bbf435 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB 2024-12-05T22:15:12,364 INFO [sync.2 {}] wal.FSHLog(777): LowReplication-Roller was enabled. 2024-12-05T22:15:12,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/.tmp/info/1b87c7e9be8b48229b92682b0d29fef6 is 1080, key is row0007/info:/1733436892963/Put/seqid=0 2024-12-05T22:15:12,371 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55452 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741866_1050] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data10]'}, localName='127.0.0.1:42417', datanodeUuid='ec5d95b1-3abc-49a7-8f6e-2970b8c20258', xmitsInProgress=0}:Exception transferring block BP-1883352144-172.17.0.2-1733436863637:blk_1073741866_1050 to mirror 127.0.0.1:45699 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:12,371 WARN [Thread-801 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1050 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45699 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:12,372 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55452 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741866_1050] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-05T22:15:12,372 WARN [Thread-801 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741866_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK], DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]) is bad. 2024-12-05T22:15:12,372 WARN [Thread-801 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741866_1050 2024-12-05T22:15:12,372 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55452 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741866_1050] {}] datanode.DataXceiver(331): 127.0.0.1:42417:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55452 dst: /127.0.0.1:42417 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:12,372 WARN [Thread-801 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK] 2024-12-05T22:15:12,377 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-05T22:15:12,377 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-05T22:15:12,377 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31b5c2bd to 127.0.0.1:62407 2024-12-05T22:15:12,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:15:12,377 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T22:15:12,378 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1262085305, stopped=false 2024-12-05T22:15:12,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741867_1051 (size=13583) 2024-12-05T22:15:12,378 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=d029b80fd32e,44177,1733436864460 2024-12-05T22:15:12,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741867_1051 (size=13583) 2024-12-05T22:15:12,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/.tmp/info/1b87c7e9be8b48229b92682b0d29fef6 2024-12-05T22:15:12,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45479-0x10184b1624b0003, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:15:12,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:15:12,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:15:12,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45479-0x10184b1624b0003, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:12,380 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-05T22:15:12,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:12,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:12,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:15:12,381 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,45395,1733436864508' ***** 2024-12-05T22:15:12,381 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-05T22:15:12,381 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,45479,1733436865872' ***** 2024-12-05T22:15:12,381 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-05T22:15:12,381 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T22:15:12,381 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T22:15:12,381 INFO [RS:1;d029b80fd32e:45479 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T22:15:12,381 INFO [RS:1;d029b80fd32e:45479 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T22:15:12,381 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,45479,1733436865872 2024-12-05T22:15:12,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:15:12,381 DEBUG [RS:1;d029b80fd32e:45479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:15:12,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45479-0x10184b1624b0003, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:15:12,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:15:12,381 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,45479,1733436865872; all regions closed. 2024-12-05T22:15:12,382 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-05T22:15:12,382 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45479,1733436865872 2024-12-05T22:15:12,382 WARN [WAL-Shutdown-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:12,383 ERROR [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1664): Shutdown / close of WAL failed: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... 2024-12-05T22:15:12,383 DEBUG [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1665): Shutdown / close exception details: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:12,383 DEBUG [RS:1;d029b80fd32e:45479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:15:12,383 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:15:12,383 INFO [RS:1;d029b80fd32e:45479 {}] hbase.ChoreService(370): Chore service for: regionserver/d029b80fd32e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-05T22:15:12,383 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T22:15:12,383 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:15:12,383 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T22:15:12,383 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T22:15:12,384 INFO [RS:1;d029b80fd32e:45479 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45479 2024-12-05T22:15:12,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45479-0x10184b1624b0003, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d029b80fd32e,45479,1733436865872 2024-12-05T22:15:12,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:15:12,385 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007f3188912b28@36f77218 rejected from java.util.concurrent.ThreadPoolExecutor@73f5faa2[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 3] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T22:15:12,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/.tmp/info/1b87c7e9be8b48229b92682b0d29fef6 as hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/info/1b87c7e9be8b48229b92682b0d29fef6 2024-12-05T22:15:12,393 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d029b80fd32e,45479,1733436865872] 2024-12-05T22:15:12,393 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing d029b80fd32e,45479,1733436865872; numProcessing=1 2024-12-05T22:15:12,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/info/1b87c7e9be8b48229b92682b0d29fef6, entries=8, sequenceid=24, filesize=13.3 K 2024-12-05T22:15:12,394 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/d029b80fd32e,45479,1733436865872 already deleted, retry=false 2024-12-05T22:15:12,394 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; d029b80fd32e,45479,1733436865872 expired; onlineServers=1 2024-12-05T22:15:12,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9682, heapSize ~10.36 KB/10608, currentSize=9.46 KB/9684 for 34c26a837ba6f5545f8521b057bbf435 in 32ms, sequenceid=24, compaction requested=false 2024-12-05T22:15:12,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 34c26a837ba6f5545f8521b057bbf435: 2024-12-05T22:15:12,396 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=23.4 K, sizeToCheck=16.0 K 2024-12-05T22:15:12,396 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:15:12,396 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/info/1b87c7e9be8b48229b92682b0d29fef6 because midkey is the same as first or last row 2024-12-05T22:15:12,396 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-05T22:15:12,396 INFO [RS:0;d029b80fd32e:45395 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T22:15:12,396 INFO [RS:0;d029b80fd32e:45395 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T22:15:12,396 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(3579): Received CLOSE for 34c26a837ba6f5545f8521b057bbf435 2024-12-05T22:15:12,396 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(3579): Received CLOSE for dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:15:12,396 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,45395,1733436864508 2024-12-05T22:15:12,396 DEBUG [RS:0;d029b80fd32e:45395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:15:12,396 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T22:15:12,396 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T22:15:12,396 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T22:15:12,397 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-05T22:15:12,397 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 34c26a837ba6f5545f8521b057bbf435, disabling compactions & flushes 2024-12-05T22:15:12,397 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:15:12,397 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-05T22:15:12,397 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:15:12,397 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 34c26a837ba6f5545f8521b057bbf435=TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435., dd5bc4db1f09996436e67228f3bee51f=hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f.} 2024-12-05T22:15:12,397 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. after waiting 0 ms 2024-12-05T22:15:12,397 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:15:12,397 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 34c26a837ba6f5545f8521b057bbf435, dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:15:12,397 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:15:12,397 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:15:12,397 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 34c26a837ba6f5545f8521b057bbf435 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB 2024-12-05T22:15:12,397 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:15:12,397 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:15:12,397 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:15:12,397 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.87 KB heapSize=5.40 KB 2024-12-05T22:15:12,398 WARN [RS_OPEN_META-regionserver/d029b80fd32e:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:12,398 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C45395%2C1733436864508.meta:.meta(num 1733436865292) roll requested 2024-12-05T22:15:12,398 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:15:12,398 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45395%2C1733436864508.meta.1733436912398.meta 2024-12-05T22:15:12,398 ERROR [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2808): ***** ABORTING region server d029b80fd32e,45395,1733436864508: Unrecoverable exception while closing hbase:meta,,1.1588230740 ***** org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:12,399 ERROR [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2815): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-05T22:15:12,401 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-05T22:15:12,403 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/.tmp/info/c600014d1db241999f58a60221a8bd9e is 1080, key is row0014/info:/1733436912364/Put/seqid=0 2024-12-05T22:15:12,403 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-05T22:15:12,403 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-05T22:15:12,403 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-05T22:15:12,403 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2819): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 259825664 }, "NonHeapMemoryUsage": { "committed": 161939456, "init": 7667712, "max": -1, "used": 160183272 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-12-05T22:15:12,404 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55484 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741868_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data10]'}, localName='127.0.0.1:42417', datanodeUuid='ec5d95b1-3abc-49a7-8f6e-2970b8c20258', xmitsInProgress=0}:Exception transferring block BP-1883352144-172.17.0.2-1733436863637:blk_1073741868_1052 to mirror 127.0.0.1:45699 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:12,405 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55484 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741868_1052] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-05T22:15:12,405 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1290984363_22 at /127.0.0.1:55484 [Receiving block BP-1883352144-172.17.0.2-1733436863637:blk_1073741868_1052] {}] datanode.DataXceiver(331): 127.0.0.1:42417:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55484 dst: /127.0.0.1:42417 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:12,405 WARN [Thread-810 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1052 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45699 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:12,405 WARN [Thread-810 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1883352144-172.17.0.2-1733436863637:blk_1073741868_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42417,DS-5540c73b-b976-4044-aa80-78188185063b,DISK], DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK]) is bad. 2024-12-05T22:15:12,405 WARN [Thread-810 {}] hdfs.DataStreamer(1850): Abandoning BP-1883352144-172.17.0.2-1733436863637:blk_1073741868_1052 2024-12-05T22:15:12,406 WARN [Thread-810 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45699,DS-f751b997-44d3-4c8b-86fe-cbb070fd6f7d,DISK] 2024-12-05T22:15:12,407 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44177 {}] master.MasterRpcServices(626): d029b80fd32e,45395,1733436864508 reported a fatal error: ***** ABORTING region server d029b80fd32e,45395,1733436864508: Unrecoverable exception while closing hbase:meta,,1.1588230740 ***** Cause: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) 2024-12-05T22:15:12,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741869_1053 (size=14663) 2024-12-05T22:15:12,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741869_1053 (size=14663) 2024-12-05T22:15:12,413 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/.tmp/info/c600014d1db241999f58a60221a8bd9e 2024-12-05T22:15:12,414 WARN [regionserver/d029b80fd32e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL 2024-12-05T22:15:12,414 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta with entries=11, filesize=3.63 KB; new WAL /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436912398.meta 2024-12-05T22:15:12,414 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38661:38661)] 2024-12-05T22:15:12,415 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta is not closed yet, will try archiving it next time 2024-12-05T22:15:12,415 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:12,415 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38977,DS-74878fdd-cd5f-4b9b-8526-a82f2386ca40,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:12,415 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta 2024-12-05T22:15:12,415 WARN [IPC Server handler 3 on default port 36083 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta has not been closed. Lease recovery is in progress. RecoveryId = 1055 for block blk_1073741834_1010 2024-12-05T22:15:12,416 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta after 1ms 2024-12-05T22:15:12,427 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/.tmp/info/c600014d1db241999f58a60221a8bd9e as hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/info/c600014d1db241999f58a60221a8bd9e 2024-12-05T22:15:12,434 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/info/c600014d1db241999f58a60221a8bd9e, entries=9, sequenceid=36, filesize=14.3 K 2024-12-05T22:15:12,436 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 34c26a837ba6f5545f8521b057bbf435 in 38ms, sequenceid=36, compaction requested=true 2024-12-05T22:15:12,441 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/data/default/TestLogRolling-testLogRollOnDatanodeDeath/34c26a837ba6f5545f8521b057bbf435/recovered.edits/39.seqid, newMaxSeqId=39, maxSeqId=1 2024-12-05T22:15:12,442 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:15:12,442 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 34c26a837ba6f5545f8521b057bbf435: 2024-12-05T22:15:12,442 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733436865961.34c26a837ba6f5545f8521b057bbf435. 2024-12-05T22:15:12,442 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing dd5bc4db1f09996436e67228f3bee51f, disabling compactions & flushes 2024-12-05T22:15:12,442 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:15:12,442 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:15:12,442 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. after waiting 0 ms 2024-12-05T22:15:12,442 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:15:12,442 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for dd5bc4db1f09996436e67228f3bee51f: 2024-12-05T22:15:12,442 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:15:12,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45479-0x10184b1624b0003, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:15:12,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45479-0x10184b1624b0003, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:15:12,493 INFO [RS:1;d029b80fd32e:45479 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,45479,1733436865872; zookeeper connection closed. 2024-12-05T22:15:12,494 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@38a58c3 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@38a58c3 2024-12-05T22:15:12,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-05T22:15:12,597 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-05T22:15:12,597 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(3579): Received CLOSE for dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing dd5bc4db1f09996436e67228f3bee51f, disabling compactions & flushes 2024-12-05T22:15:12,598 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:15:12,598 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. after waiting 0 ms 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for dd5bc4db1f09996436e67228f3bee51f: 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733436865342.dd5bc4db1f09996436e67228f3bee51f. 2024-12-05T22:15:12,598 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:meta,,1.1588230740 2024-12-05T22:15:12,602 DEBUG [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, dd5bc4db1f09996436e67228f3bee51f 2024-12-05T22:15:12,764 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436888944 to hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/oldWALs/d029b80fd32e%2C45395%2C1733436864508.1733436888944 2024-12-05T22:15:12,765 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436892958 to hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/oldWALs/d029b80fd32e%2C45395%2C1733436864508.1733436892958 2024-12-05T22:15:12,766 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.1733436892983 to hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/oldWALs/d029b80fd32e%2C45395%2C1733436864508.1733436892983 2024-12-05T22:15:12,770 INFO [regionserver/d029b80fd32e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:15:12,777 INFO [regionserver/d029b80fd32e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T22:15:12,777 INFO [regionserver/d029b80fd32e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T22:15:12,802 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1624): We were exiting though online regions are not empty, because some regions failed closing 2024-12-05T22:15:12,802 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,45395,1733436864508; all regions closed. 2024-12-05T22:15:12,802 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508 2024-12-05T22:15:12,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741870_1054 (size=93) 2024-12-05T22:15:12,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741870_1054 (size=93) 2024-12-05T22:15:13,677 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6203ba3[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741829_1005 to 127.0.0.1:45699 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:13,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:15:13,701 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@7e694f64 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1883352144-172.17.0.2-1733436863637:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:38977,null,null]) java.net.ConnectException: Call From d029b80fd32e/172.17.0.2 to localhost:38899 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-05T22:15:13,926 INFO [regionserver/d029b80fd32e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:15:14,677 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@580ffa77[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40967, datanodeUuid=730af7b4-0272-418a-88c0-5cf7aea64c4c, infoPort=38661, infoSecurePort=0, ipcPort=41489, storageInfo=lv=-57;cid=testClusterID;nsid=656522824;c=1733436863637):Failed to transfer BP-1883352144-172.17.0.2-1733436863637:blk_1073741825_1001 to 127.0.0.1:45699 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:14,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741838_1014 (size=76) 2024-12-05T22:15:15,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741830_1047 (size=47148) 2024-12-05T22:15:15,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741859_1042 (size=1618) 2024-12-05T22:15:16,417 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta after 4002ms 2024-12-05T22:15:17,443 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,454 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,455 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,457 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,457 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,457 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,805 ERROR [WAL-Shutdown-0 {}] wal.FSHLog(508): We have waited 5 seconds but the close of writer(s) doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-12-05T22:15:17,805 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508 2024-12-05T22:15:17,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741865_1049 (size=13514) 2024-12-05T22:15:17,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741865_1049 (size=13514) 2024-12-05T22:15:17,809 DEBUG [RS:0;d029b80fd32e:45395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:15:17,809 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:15:17,809 INFO [RS:0;d029b80fd32e:45395 {}] hbase.ChoreService(370): Chore service for: regionserver/d029b80fd32e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-05T22:15:17,809 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:15:17,810 INFO [RS:0;d029b80fd32e:45395 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45395 2024-12-05T22:15:17,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d029b80fd32e,45395,1733436864508 2024-12-05T22:15:17,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:15:17,814 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d029b80fd32e,45395,1733436864508] 2024-12-05T22:15:17,814 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing d029b80fd32e,45395,1733436864508; numProcessing=2 2024-12-05T22:15:17,815 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/d029b80fd32e,45395,1733436864508 already deleted, retry=false 2024-12-05T22:15:17,815 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; d029b80fd32e,45395,1733436864508 expired; onlineServers=0 2024-12-05T22:15:17,816 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,44177,1733436864460' ***** 2024-12-05T22:15:17,816 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T22:15:17,816 DEBUG [M:0;d029b80fd32e:44177 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ee65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:15:17,816 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,44177,1733436864460 2024-12-05T22:15:17,816 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,44177,1733436864460; all regions closed. 2024-12-05T22:15:17,816 DEBUG [M:0;d029b80fd32e:44177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:15:17,816 DEBUG [M:0;d029b80fd32e:44177 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T22:15:17,816 DEBUG [M:0;d029b80fd32e:44177 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T22:15:17,816 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T22:15:17,816 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436864678 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436864678,5,FailOnTimeoutGroup] 2024-12-05T22:15:17,816 INFO [M:0;d029b80fd32e:44177 {}] hbase.ChoreService(370): Chore service for: master/d029b80fd32e:0 had [] on shutdown 2024-12-05T22:15:17,816 DEBUG [M:0;d029b80fd32e:44177 {}] master.HMaster(1733): Stopping service threads 2024-12-05T22:15:17,817 INFO [M:0;d029b80fd32e:44177 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T22:15:17,817 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436864680 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436864680,5,FailOnTimeoutGroup] 2024-12-05T22:15:17,817 INFO [M:0;d029b80fd32e:44177 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T22:15:17,817 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T22:15:17,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T22:15:17,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:17,818 DEBUG [M:0;d029b80fd32e:44177 {}] zookeeper.ZKUtil(347): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T22:15:17,818 WARN [M:0;d029b80fd32e:44177 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T22:15:17,818 INFO [M:0;d029b80fd32e:44177 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-05T22:15:17,818 INFO [M:0;d029b80fd32e:44177 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T22:15:17,818 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:15:17,818 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:15:17,818 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:15:17,818 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:15:17,818 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:15:17,818 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:15:17,818 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.08 KB heapSize=49.29 KB 2024-12-05T22:15:17,842 DEBUG [M:0;d029b80fd32e:44177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/614503f1e6a84d66b16649c845084f89 is 82, key is hbase:meta,,1/info:regioninfo/1733436865317/Put/seqid=0 2024-12-05T22:15:17,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741871_1056 (size=5672) 2024-12-05T22:15:17,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741871_1056 (size=5672) 2024-12-05T22:15:17,849 INFO [M:0;d029b80fd32e:44177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/614503f1e6a84d66b16649c845084f89 2024-12-05T22:15:17,879 DEBUG [M:0;d029b80fd32e:44177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b9684976f6e94d918d8785f6f8a62192 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733436866344/Put/seqid=0 2024-12-05T22:15:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741872_1057 (size=7465) 2024-12-05T22:15:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741872_1057 (size=7465) 2024-12-05T22:15:17,886 INFO [M:0;d029b80fd32e:44177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.41 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b9684976f6e94d918d8785f6f8a62192 2024-12-05T22:15:17,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:15:17,914 INFO [RS:0;d029b80fd32e:45395 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,45395,1733436864508; zookeeper connection closed. 2024-12-05T22:15:17,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45395-0x10184b1624b0001, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:15:17,915 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@139a5ba4 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@139a5ba4 2024-12-05T22:15:17,915 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-05T22:15:17,915 DEBUG [M:0;d029b80fd32e:44177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/555e21f05ba640358b63e7973176ae74 is 69, key is d029b80fd32e,45395,1733436864508/rs:state/1733436864752/Put/seqid=0 2024-12-05T22:15:17,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741873_1058 (size=5224) 2024-12-05T22:15:17,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741873_1058 (size=5224) 2024-12-05T22:15:17,923 INFO [M:0;d029b80fd32e:44177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/555e21f05ba640358b63e7973176ae74 2024-12-05T22:15:17,944 DEBUG [M:0;d029b80fd32e:44177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/260a68ce3c15474e924ba86f8bc1c1ec is 52, key is load_balancer_on/state:d/1733436865855/Put/seqid=0 2024-12-05T22:15:17,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741874_1059 (size=5056) 2024-12-05T22:15:17,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741874_1059 (size=5056) 2024-12-05T22:15:17,950 INFO [M:0;d029b80fd32e:44177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/260a68ce3c15474e924ba86f8bc1c1ec 2024-12-05T22:15:17,956 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/614503f1e6a84d66b16649c845084f89 as hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/614503f1e6a84d66b16649c845084f89 2024-12-05T22:15:17,962 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/614503f1e6a84d66b16649c845084f89, entries=8, sequenceid=97, filesize=5.5 K 2024-12-05T22:15:17,963 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b9684976f6e94d918d8785f6f8a62192 as hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b9684976f6e94d918d8785f6f8a62192 2024-12-05T22:15:17,965 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:15:17,966 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,969 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b9684976f6e94d918d8785f6f8a62192, entries=11, sequenceid=97, filesize=7.3 K 2024-12-05T22:15:17,971 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/555e21f05ba640358b63e7973176ae74 as hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/555e21f05ba640358b63e7973176ae74 2024-12-05T22:15:17,977 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/555e21f05ba640358b63e7973176ae74, entries=2, sequenceid=97, filesize=5.1 K 2024-12-05T22:15:17,978 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/260a68ce3c15474e924ba86f8bc1c1ec as hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/260a68ce3c15474e924ba86f8bc1c1ec 2024-12-05T22:15:17,982 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,983 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,984 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/260a68ce3c15474e924ba86f8bc1c1ec, entries=1, sequenceid=97, filesize=4.9 K 2024-12-05T22:15:17,985 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.08 KB/41039, heapSize ~49.23 KB/50408, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 167ms, sequenceid=97, compaction requested=false 2024-12-05T22:15:17,985 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,986 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,986 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:17,987 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:15:17,987 DEBUG [M:0;d029b80fd32e:44177 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:15:17,988 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/MasterData/WALs/d029b80fd32e,44177,1733436864460 2024-12-05T22:15:17,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42417 is added to blk_1073741863_1046 (size=757) 2024-12-05T22:15:17,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40967 is added to blk_1073741863_1046 (size=757) 2024-12-05T22:15:17,992 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:15:17,992 INFO [M:0;d029b80fd32e:44177 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-05T22:15:17,993 INFO [M:0;d029b80fd32e:44177 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:44177 2024-12-05T22:15:17,996 DEBUG [M:0;d029b80fd32e:44177 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/d029b80fd32e,44177,1733436864460 already deleted, retry=false 2024-12-05T22:15:18,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:15:18,098 INFO [M:0;d029b80fd32e:44177 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,44177,1733436864460; zookeeper connection closed. 2024-12-05T22:15:18,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44177-0x10184b1624b0000, quorum=127.0.0.1:62407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:15:18,101 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2aec2b59{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:18,102 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a79e9ef{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:15:18,102 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:15:18,102 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3295b233{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:15:18,102 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d021301{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,STOPPED} 2024-12-05T22:15:18,104 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:15:18,104 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:15:18,104 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:15:18,104 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1883352144-172.17.0.2-1733436863637 (Datanode Uuid 730af7b4-0272-418a-88c0-5cf7aea64c4c) service to localhost/127.0.0.1:36083 2024-12-05T22:15:18,103 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@9832f50 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:38977,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:38899 , LocalHost:localPort d029b80fd32e/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-05T22:15:18,104 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@9832f50 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:40967,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1883352144-172.17.0.2-1733436863637 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:18,104 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@9832f50 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:38977,null,null], DatanodeInfoWithStorage[127.0.0.1:40967,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1883352144-172.17.0.2-1733436863637:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:38977,null,null], DatanodeInfoWithStorage[127.0.0.1:40967,null,null]] 2024-12-05T22:15:18,105 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data3/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:18,105 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data4/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:18,105 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:15:18,107 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@560986a2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:18,108 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1e28d6de{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:15:18,108 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:15:18,108 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@612cf16{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:15:18,108 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40616806{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,STOPPED} 2024-12-05T22:15:18,109 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:15:18,109 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:15:18,109 WARN [BP-1883352144-172.17.0.2-1733436863637 heartbeating to localhost/127.0.0.1:36083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1883352144-172.17.0.2-1733436863637 (Datanode Uuid ec5d95b1-3abc-49a7-8f6e-2970b8c20258) service to localhost/127.0.0.1:36083 2024-12-05T22:15:18,109 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:15:18,110 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data10/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:18,111 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:15:18,112 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/cluster_db52a60d-a703-5081-64f0-6fc298c2b9bb/dfs/data/data9/current/BP-1883352144-172.17.0.2-1733436863637 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:18,119 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7ef935fd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:15:18,120 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3ac35248{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:15:18,120 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:15:18,120 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52ca9eab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:15:18,120 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c5ad4dc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir/,STOPPED} 2024-12-05T22:15:18,129 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-05T22:15:18,172 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-05T22:15:18,180 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=86 (was 64) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:36083 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36083 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:36083 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36083 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:36083 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:36083 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-15-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36083 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$$Lambda$793/0x00007f3188b871f8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:36083 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-14-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:36083 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=424 (was 401) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=4302 (was 3639) - AvailableMemoryMB LEAK? - 2024-12-05T22:15:18,187 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=86, OpenFileDescriptor=424, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=4302 2024-12-05T22:15:18,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T22:15:18,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.log.dir so I do NOT create it in target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa 2024-12-05T22:15:18,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/a909c65a-ed86-28a5-434b-ad77ed733810/hadoop.tmp.dir so I do NOT create it in target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa 2024-12-05T22:15:18,187 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20, deleteOnExit=true 2024-12-05T22:15:18,187 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-05T22:15:18,188 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/test.cache.data in system properties and HBase conf 2024-12-05T22:15:18,188 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T22:15:18,188 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir in system properties and HBase conf 2024-12-05T22:15:18,188 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T22:15:18,188 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T22:15:18,188 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-05T22:15:18,188 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T22:15:18,189 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/nfs.dump.dir in system properties and HBase conf 2024-12-05T22:15:18,190 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/java.io.tmpdir in system properties and HBase conf 2024-12-05T22:15:18,190 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:15:18,190 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T22:15:18,190 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T22:15:18,210 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:15:18,304 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:15:18,311 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:15:18,317 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:15:18,317 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:15:18,317 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:15:18,318 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:15:18,319 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3338083e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:15:18,319 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47516fb1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:15:18,418 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:18,469 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@393bcdc0{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/java.io.tmpdir/jetty-localhost-38353-hadoop-hdfs-3_4_1-tests_jar-_-any-14281715607322333954/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:15:18,469 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15fa4d7a{HTTP/1.1, (http/1.1)}{localhost:38353} 2024-12-05T22:15:18,469 INFO [Time-limited test {}] server.Server(415): Started @172187ms 2024-12-05T22:15:18,483 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:15:18,553 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:15:18,558 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:15:18,559 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:15:18,559 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:15:18,559 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:15:18,559 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3436fcc3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:15:18,560 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3dddc2f1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:15:18,685 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6997e634{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/java.io.tmpdir/jetty-localhost-38039-hadoop-hdfs-3_4_1-tests_jar-_-any-12218216829271324919/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:18,685 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3479543b{HTTP/1.1, (http/1.1)}{localhost:38039} 2024-12-05T22:15:18,685 INFO [Time-limited test {}] server.Server(415): Started @172403ms 2024-12-05T22:15:18,687 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:15:18,719 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:15:18,722 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:15:18,722 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:15:18,722 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:15:18,722 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:15:18,723 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d286f08{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:15:18,723 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6058ad7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:15:18,807 WARN [Thread-932 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data2/current/BP-1259869556-172.17.0.2-1733436918234/current, will proceed with Du for space computation calculation, 2024-12-05T22:15:18,807 WARN [Thread-931 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data1/current/BP-1259869556-172.17.0.2-1733436918234/current, will proceed with Du for space computation calculation, 2024-12-05T22:15:18,837 WARN [Thread-910 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:15:18,840 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa16bf33e1bf184f1 with lease ID 0x7b4142fca855b642: Processing first storage report for DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7 from datanode DatanodeRegistration(127.0.0.1:37065, datanodeUuid=4eb9d08a-9fc2-411d-9fb2-656977000172, infoPort=41009, infoSecurePort=0, ipcPort=44049, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234) 2024-12-05T22:15:18,840 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa16bf33e1bf184f1 with lease ID 0x7b4142fca855b642: from storage DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7 node DatanodeRegistration(127.0.0.1:37065, datanodeUuid=4eb9d08a-9fc2-411d-9fb2-656977000172, infoPort=41009, infoSecurePort=0, ipcPort=44049, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:18,840 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa16bf33e1bf184f1 with lease ID 0x7b4142fca855b642: Processing first storage report for DS-8611fc5e-3e31-4abc-a2bf-ce8e07538000 from datanode DatanodeRegistration(127.0.0.1:37065, datanodeUuid=4eb9d08a-9fc2-411d-9fb2-656977000172, infoPort=41009, infoSecurePort=0, ipcPort=44049, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234) 2024-12-05T22:15:18,840 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa16bf33e1bf184f1 with lease ID 0x7b4142fca855b642: from storage DS-8611fc5e-3e31-4abc-a2bf-ce8e07538000 node DatanodeRegistration(127.0.0.1:37065, datanodeUuid=4eb9d08a-9fc2-411d-9fb2-656977000172, infoPort=41009, infoSecurePort=0, ipcPort=44049, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:18,866 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bfb26c3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/java.io.tmpdir/jetty-localhost-37237-hadoop-hdfs-3_4_1-tests_jar-_-any-17173938201836706611/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:18,867 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5f202bff{HTTP/1.1, (http/1.1)}{localhost:37237} 2024-12-05T22:15:18,867 INFO [Time-limited test {}] server.Server(415): Started @172584ms 2024-12-05T22:15:18,868 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:15:18,973 WARN [Thread-957 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data3/current/BP-1259869556-172.17.0.2-1733436918234/current, will proceed with Du for space computation calculation, 2024-12-05T22:15:18,973 WARN [Thread-958 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data4/current/BP-1259869556-172.17.0.2-1733436918234/current, will proceed with Du for space computation calculation, 2024-12-05T22:15:18,997 WARN [Thread-946 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:15:19,001 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x77b5202599f67ad5 with lease ID 0x7b4142fca855b643: Processing first storage report for DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d from datanode DatanodeRegistration(127.0.0.1:42555, datanodeUuid=0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1, infoPort=32779, infoSecurePort=0, ipcPort=40451, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234) 2024-12-05T22:15:19,001 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x77b5202599f67ad5 with lease ID 0x7b4142fca855b643: from storage DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d node DatanodeRegistration(127.0.0.1:42555, datanodeUuid=0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1, infoPort=32779, infoSecurePort=0, ipcPort=40451, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:19,001 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x77b5202599f67ad5 with lease ID 0x7b4142fca855b643: Processing first storage report for DS-a53518c1-f2e4-427a-93fb-a64e353ce496 from datanode DatanodeRegistration(127.0.0.1:42555, datanodeUuid=0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1, infoPort=32779, infoSecurePort=0, ipcPort=40451, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234) 2024-12-05T22:15:19,001 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x77b5202599f67ad5 with lease ID 0x7b4142fca855b643: from storage DS-a53518c1-f2e4-427a-93fb-a64e353ce496 node DatanodeRegistration(127.0.0.1:42555, datanodeUuid=0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1, infoPort=32779, infoSecurePort=0, ipcPort=40451, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:19,104 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa 2024-12-05T22:15:19,107 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/zookeeper_0, clientPort=51580, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T22:15:19,108 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=51580 2024-12-05T22:15:19,108 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:15:19,110 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:15:19,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:15:19,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:15:19,122 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba with version=8 2024-12-05T22:15:19,122 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/hbase-staging 2024-12-05T22:15:19,124 INFO [Time-limited test {}] client.ConnectionUtils(129): master/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:15:19,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:15:19,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:15:19,124 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:15:19,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:15:19,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:15:19,125 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:15:19,125 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:15:19,127 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:46117 2024-12-05T22:15:19,127 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:15:19,129 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:15:19,133 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:46117 connecting to ZooKeeper ensemble=127.0.0.1:51580 2024-12-05T22:15:19,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:461170x0, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:15:19,142 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46117-0x10184b237d20000 connected 2024-12-05T22:15:19,162 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:15:19,163 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:15:19,163 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:15:19,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46117 2024-12-05T22:15:19,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46117 2024-12-05T22:15:19,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46117 2024-12-05T22:15:19,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46117 2024-12-05T22:15:19,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46117 2024-12-05T22:15:19,166 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba, hbase.cluster.distributed=false 2024-12-05T22:15:19,182 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:15:19,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:15:19,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:15:19,182 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:15:19,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:15:19,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:15:19,182 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:15:19,182 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:15:19,183 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38731 2024-12-05T22:15:19,183 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T22:15:19,184 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T22:15:19,185 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:15:19,186 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:15:19,188 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:38731 connecting to ZooKeeper ensemble=127.0.0.1:51580 2024-12-05T22:15:19,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387310x0, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:15:19,191 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387310x0, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:15:19,191 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38731-0x10184b237d20001 connected 2024-12-05T22:15:19,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:15:19,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:15:19,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38731 2024-12-05T22:15:19,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38731 2024-12-05T22:15:19,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38731 2024-12-05T22:15:19,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38731 2024-12-05T22:15:19,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38731 2024-12-05T22:15:19,194 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/d029b80fd32e,46117,1733436919123 2024-12-05T22:15:19,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:15:19,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:15:19,199 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d029b80fd32e,46117,1733436919123 2024-12-05T22:15:19,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:15:19,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:15:19,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,203 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:15:19,203 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:15:19,203 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d029b80fd32e,46117,1733436919123 from backup master directory 2024-12-05T22:15:19,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d029b80fd32e,46117,1733436919123 2024-12-05T22:15:19,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:15:19,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:15:19,206 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:15:19,207 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d029b80fd32e,46117,1733436919123 2024-12-05T22:15:19,207 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d029b80fd32e:46117 2024-12-05T22:15:19,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:15:19,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:15:19,220 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/hbase.id with ID: 8b763724-4521-42a7-89dc-e8151b7caebf 2024-12-05T22:15:19,232 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:15:19,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:15:19,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:15:19,244 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:15:19,245 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T22:15:19,245 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:15:19,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:15:19,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:15:19,255 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store 2024-12-05T22:15:19,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:15:19,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:15:19,263 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:15:19,263 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:15:19,263 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:15:19,263 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:15:19,263 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:15:19,263 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:15:19,263 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:15:19,263 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:15:19,264 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/.initializing 2024-12-05T22:15:19,264 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123 2024-12-05T22:15:19,267 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C46117%2C1733436919123, suffix=, logDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123, archiveDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/oldWALs, maxLogs=10 2024-12-05T22:15:19,268 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C46117%2C1733436919123.1733436919268 2024-12-05T22:15:19,273 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123/d029b80fd32e%2C46117%2C1733436919123.1733436919268 2024-12-05T22:15:19,273 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:32779:32779)] 2024-12-05T22:15:19,273 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:15:19,273 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:15:19,273 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:15:19,273 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:15:19,275 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:15:19,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T22:15:19,276 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:15:19,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:15:19,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T22:15:19,278 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:15:19,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:15:19,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T22:15:19,280 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:15:19,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:15:19,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T22:15:19,281 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,282 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:15:19,283 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:15:19,283 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:15:19,285 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T22:15:19,286 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:15:19,287 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:15:19,288 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=835799, jitterRate=0.06277438998222351}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T22:15:19,288 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:15:19,289 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T22:15:19,292 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f7cc40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:15:19,293 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-05T22:15:19,293 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T22:15:19,293 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T22:15:19,293 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T22:15:19,293 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T22:15:19,294 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-05T22:15:19,294 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T22:15:19,296 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T22:15:19,296 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T22:15:19,298 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-05T22:15:19,298 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T22:15:19,299 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T22:15:19,299 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-05T22:15:19,300 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T22:15:19,300 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T22:15:19,303 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-05T22:15:19,303 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T22:15:19,304 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T22:15:19,306 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T22:15:19,306 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T22:15:19,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:15:19,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:15:19,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,309 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=d029b80fd32e,46117,1733436919123, sessionid=0x10184b237d20000, setting cluster-up flag (Was=false) 2024-12-05T22:15:19,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,318 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T22:15:19,319 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,46117,1733436919123 2024-12-05T22:15:19,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,326 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T22:15:19,327 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,46117,1733436919123 2024-12-05T22:15:19,329 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-05T22:15:19,329 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-05T22:15:19,330 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T22:15:19,330 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d029b80fd32e,46117,1733436919123 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T22:15:19,330 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:15:19,330 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:15:19,330 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:15:19,330 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:15:19,330 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d029b80fd32e:0, corePoolSize=10, maxPoolSize=10 2024-12-05T22:15:19,330 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,330 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:15:19,330 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,331 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733436949331 2024-12-05T22:15:19,331 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T22:15:19,331 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T22:15:19,331 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T22:15:19,331 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T22:15:19,331 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T22:15:19,332 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T22:15:19,332 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:15:19,332 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-05T22:15:19,332 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,332 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T22:15:19,332 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T22:15:19,332 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T22:15:19,333 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T22:15:19,333 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T22:15:19,333 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436919333,5,FailOnTimeoutGroup] 2024-12-05T22:15:19,333 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436919333,5,FailOnTimeoutGroup] 2024-12-05T22:15:19,333 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,333 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,333 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T22:15:19,333 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,333 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,333 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:15:19,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:15:19,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:15:19,341 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-05T22:15:19,341 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba 2024-12-05T22:15:19,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:15:19,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:15:19,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:15:19,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:15:19,351 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:15:19,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:15:19,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:15:19,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:15:19,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:15:19,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:15:19,355 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:15:19,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:15:19,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/hbase/meta/1588230740 2024-12-05T22:15:19,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/hbase/meta/1588230740 2024-12-05T22:15:19,357 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:15:19,359 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:15:19,361 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:15:19,361 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=818451, jitterRate=0.04071468114852905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:15:19,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:15:19,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:15:19,361 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:15:19,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:15:19,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:15:19,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:15:19,362 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:15:19,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:15:19,363 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:15:19,363 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-05T22:15:19,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T22:15:19,364 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T22:15:19,365 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T22:15:19,406 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d029b80fd32e:38731 2024-12-05T22:15:19,407 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1008): ClusterId : 8b763724-4521-42a7-89dc-e8151b7caebf 2024-12-05T22:15:19,407 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T22:15:19,409 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T22:15:19,409 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T22:15:19,410 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T22:15:19,410 DEBUG [RS:0;d029b80fd32e:38731 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7671a9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:15:19,411 DEBUG [RS:0;d029b80fd32e:38731 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60f61bb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:15:19,411 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-05T22:15:19,411 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-05T22:15:19,411 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-05T22:15:19,412 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(3073): reportForDuty to master=d029b80fd32e,46117,1733436919123 with isa=d029b80fd32e/172.17.0.2:38731, startcode=1733436919181 2024-12-05T22:15:19,412 DEBUG [RS:0;d029b80fd32e:38731 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:15:19,414 INFO [RS-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47471, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:15:19,415 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46117 {}] master.ServerManager(332): Checking decommissioned status of RegionServer d029b80fd32e,38731,1733436919181 2024-12-05T22:15:19,415 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46117 {}] master.ServerManager(486): Registering regionserver=d029b80fd32e,38731,1733436919181 2024-12-05T22:15:19,417 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba 2024-12-05T22:15:19,417 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:38283 2024-12-05T22:15:19,417 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-05T22:15:19,419 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:19,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:15:19,420 DEBUG [RS:0;d029b80fd32e:38731 {}] zookeeper.ZKUtil(111): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d029b80fd32e,38731,1733436919181 2024-12-05T22:15:19,420 WARN [RS:0;d029b80fd32e:38731 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:15:19,420 INFO [RS:0;d029b80fd32e:38731 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:15:19,420 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181 2024-12-05T22:15:19,420 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d029b80fd32e,38731,1733436919181] 2024-12-05T22:15:19,423 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-05T22:15:19,423 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T22:15:19,426 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T22:15:19,426 INFO [RS:0;d029b80fd32e:38731 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T22:15:19,426 INFO [RS:0;d029b80fd32e:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,426 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-05T22:15:19,427 INFO [RS:0;d029b80fd32e:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,427 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,427 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,427 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,427 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,428 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,428 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:15:19,428 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,428 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,428 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,428 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,428 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:15:19,428 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:15:19,428 DEBUG [RS:0;d029b80fd32e:38731 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:15:19,430 INFO [RS:0;d029b80fd32e:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,430 INFO [RS:0;d029b80fd32e:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,430 INFO [RS:0;d029b80fd32e:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,430 INFO [RS:0;d029b80fd32e:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,430 INFO [RS:0;d029b80fd32e:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,38731,1733436919181-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:15:19,448 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T22:15:19,448 INFO [RS:0;d029b80fd32e:38731 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,38731,1733436919181-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,463 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.Replication(204): d029b80fd32e,38731,1733436919181 started 2024-12-05T22:15:19,463 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1767): Serving as d029b80fd32e,38731,1733436919181, RpcServer on d029b80fd32e/172.17.0.2:38731, sessionid=0x10184b237d20001 2024-12-05T22:15:19,463 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T22:15:19,463 DEBUG [RS:0;d029b80fd32e:38731 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d029b80fd32e,38731,1733436919181 2024-12-05T22:15:19,463 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,38731,1733436919181' 2024-12-05T22:15:19,463 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T22:15:19,463 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T22:15:19,464 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T22:15:19,464 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T22:15:19,464 DEBUG [RS:0;d029b80fd32e:38731 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d029b80fd32e,38731,1733436919181 2024-12-05T22:15:19,464 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,38731,1733436919181' 2024-12-05T22:15:19,464 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T22:15:19,464 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T22:15:19,465 DEBUG [RS:0;d029b80fd32e:38731 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T22:15:19,465 INFO [RS:0;d029b80fd32e:38731 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T22:15:19,465 INFO [RS:0;d029b80fd32e:38731 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T22:15:19,515 WARN [d029b80fd32e:46117 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-05T22:15:19,567 INFO [RS:0;d029b80fd32e:38731 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C38731%2C1733436919181, suffix=, logDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181, archiveDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/oldWALs, maxLogs=32 2024-12-05T22:15:19,568 INFO [RS:0;d029b80fd32e:38731 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38731%2C1733436919181.1733436919568 2024-12-05T22:15:19,574 INFO [RS:0;d029b80fd32e:38731 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 2024-12-05T22:15:19,574 DEBUG [RS:0;d029b80fd32e:38731 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32779:32779),(127.0.0.1/127.0.0.1:41009:41009)] 2024-12-05T22:15:19,765 DEBUG [d029b80fd32e:46117 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T22:15:19,766 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d029b80fd32e,38731,1733436919181 2024-12-05T22:15:19,767 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,38731,1733436919181, state=OPENING 2024-12-05T22:15:19,769 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T22:15:19,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:19,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=d029b80fd32e,38731,1733436919181}] 2024-12-05T22:15:19,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:15:19,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:15:19,924 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d029b80fd32e,38731,1733436919181 2024-12-05T22:15:19,924 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T22:15:19,926 INFO [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60856, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T22:15:19,930 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-05T22:15:19,930 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:15:19,932 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C38731%2C1733436919181.meta, suffix=.meta, logDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181, archiveDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/oldWALs, maxLogs=32 2024-12-05T22:15:19,933 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38731%2C1733436919181.meta.1733436919933.meta 2024-12-05T22:15:19,938 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.meta.1733436919933.meta 2024-12-05T22:15:19,938 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009),(127.0.0.1/127.0.0.1:32779:32779)] 2024-12-05T22:15:19,938 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:15:19,938 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T22:15:19,938 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T22:15:19,938 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T22:15:19,938 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T22:15:19,939 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:15:19,939 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-05T22:15:19,939 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-05T22:15:19,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:15:19,941 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:15:19,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:15:19,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:15:19,942 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:15:19,942 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:15:19,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:15:19,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:15:19,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:15:19,945 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/hbase/meta/1588230740 2024-12-05T22:15:19,946 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/hbase/meta/1588230740 2024-12-05T22:15:19,947 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:15:19,949 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:15:19,950 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881741, jitterRate=0.12119241058826447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:15:19,950 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:15:19,951 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733436919924 2024-12-05T22:15:19,953 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T22:15:19,953 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-05T22:15:19,953 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,38731,1733436919181 2024-12-05T22:15:19,954 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,38731,1733436919181, state=OPEN 2024-12-05T22:15:19,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:15:19,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:15:19,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:15:19,961 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:15:19,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T22:15:19,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=d029b80fd32e,38731,1733436919181 in 189 msec 2024-12-05T22:15:19,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T22:15:19,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 600 msec 2024-12-05T22:15:19,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 637 msec 2024-12-05T22:15:19,967 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733436919967, completionTime=-1 2024-12-05T22:15:19,967 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T22:15:19,967 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-05T22:15:19,968 DEBUG [hconnection-0x44943e08-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:15:19,969 INFO [RS-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60870, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:15:19,970 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-05T22:15:19,970 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733436979970 2024-12-05T22:15:19,970 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733437039970 2024-12-05T22:15:19,970 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec 2024-12-05T22:15:19,976 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,46117,1733436919123-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,976 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,46117,1733436919123-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,976 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,46117,1733436919123-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,976 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d029b80fd32e:46117, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,976 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:19,976 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-05T22:15:19,977 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:15:19,978 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-05T22:15:19,978 DEBUG [master/d029b80fd32e:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-05T22:15:19,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:15:19,979 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:19,980 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:15:19,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:15:19,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:15:19,989 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9e2824609f96a6001d8013bb5c6d5a69, NAME => 'hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba 2024-12-05T22:15:19,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:15:19,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:15:19,995 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:15:19,995 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 9e2824609f96a6001d8013bb5c6d5a69, disabling compactions & flushes 2024-12-05T22:15:19,995 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:15:19,995 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:15:19,995 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. after waiting 0 ms 2024-12-05T22:15:19,995 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:15:19,995 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:15:19,995 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9e2824609f96a6001d8013bb5c6d5a69: 2024-12-05T22:15:19,996 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:15:19,997 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733436919997"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733436919997"}]},"ts":"1733436919997"} 2024-12-05T22:15:19,999 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:15:19,999 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:15:20,000 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436919999"}]},"ts":"1733436919999"} 2024-12-05T22:15:20,001 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-05T22:15:20,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=9e2824609f96a6001d8013bb5c6d5a69, ASSIGN}] 2024-12-05T22:15:20,008 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=9e2824609f96a6001d8013bb5c6d5a69, ASSIGN 2024-12-05T22:15:20,009 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=9e2824609f96a6001d8013bb5c6d5a69, ASSIGN; state=OFFLINE, location=d029b80fd32e,38731,1733436919181; forceNewPlan=false, retain=false 2024-12-05T22:15:20,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=9e2824609f96a6001d8013bb5c6d5a69, regionState=OPENING, regionLocation=d029b80fd32e,38731,1733436919181 2024-12-05T22:15:20,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 9e2824609f96a6001d8013bb5c6d5a69, server=d029b80fd32e,38731,1733436919181}] 2024-12-05T22:15:20,315 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d029b80fd32e,38731,1733436919181 2024-12-05T22:15:20,319 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:15:20,319 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 9e2824609f96a6001d8013bb5c6d5a69, NAME => 'hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:15:20,319 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:15:20,319 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:15:20,319 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:15:20,320 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:15:20,321 INFO [StoreOpener-9e2824609f96a6001d8013bb5c6d5a69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:15:20,323 INFO [StoreOpener-9e2824609f96a6001d8013bb5c6d5a69-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e2824609f96a6001d8013bb5c6d5a69 columnFamilyName info 2024-12-05T22:15:20,323 DEBUG [StoreOpener-9e2824609f96a6001d8013bb5c6d5a69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:20,323 INFO [StoreOpener-9e2824609f96a6001d8013bb5c6d5a69-1 {}] regionserver.HStore(327): Store=9e2824609f96a6001d8013bb5c6d5a69/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:15:20,324 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/hbase/namespace/9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:15:20,325 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/hbase/namespace/9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:15:20,327 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:15:20,329 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/hbase/namespace/9e2824609f96a6001d8013bb5c6d5a69/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:15:20,329 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 9e2824609f96a6001d8013bb5c6d5a69; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=799239, jitterRate=0.01628577709197998}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:15:20,330 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 9e2824609f96a6001d8013bb5c6d5a69: 2024-12-05T22:15:20,330 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69., pid=6, masterSystemTime=1733436920315 2024-12-05T22:15:20,333 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:15:20,333 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:15:20,333 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=9e2824609f96a6001d8013bb5c6d5a69, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,38731,1733436919181 2024-12-05T22:15:20,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T22:15:20,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 9e2824609f96a6001d8013bb5c6d5a69, server=d029b80fd32e,38731,1733436919181 in 173 msec 2024-12-05T22:15:20,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T22:15:20,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=9e2824609f96a6001d8013bb5c6d5a69, ASSIGN in 332 msec 2024-12-05T22:15:20,341 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:15:20,342 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436920341"}]},"ts":"1733436920341"} 2024-12-05T22:15:20,343 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-05T22:15:20,346 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:15:20,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 369 msec 2024-12-05T22:15:20,379 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-05T22:15:20,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:15:20,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:20,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:15:20,389 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-05T22:15:20,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:15:20,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 12 msec 2024-12-05T22:15:20,411 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-05T22:15:20,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:15:20,419 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:20,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 11 msec 2024-12-05T22:15:20,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-05T22:15:20,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-05T22:15:20,441 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.234sec 2024-12-05T22:15:20,441 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T22:15:20,441 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T22:15:20,441 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T22:15:20,441 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T22:15:20,441 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T22:15:20,441 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,46117,1733436919123-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:15:20,442 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,46117,1733436919123-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T22:15:20,443 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-05T22:15:20,443 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T22:15:20,443 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,46117,1733436919123-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:15:20,497 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x545a7404 to 127.0.0.1:51580 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68caed40 2024-12-05T22:15:20,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784500cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:15:20,503 DEBUG [hconnection-0x475a5269-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:15:20,506 INFO [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60872, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:15:20,507 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=d029b80fd32e,46117,1733436919123 2024-12-05T22:15:20,508 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:15:20,512 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-05T22:15:20,512 INFO [Time-limited test {}] wal.TestLogRolling(297): Starting testLogRollOnPipelineRestart 2024-12-05T22:15:20,512 INFO [Time-limited test {}] wal.TestLogRolling(300): Replication=2 2024-12-05T22:15:20,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T22:15:20,516 INFO [RS-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58808, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T22:15:20,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46117 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-05T22:15:20,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46117 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-05T22:15:20,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46117 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:15:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46117 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-12-05T22:15:20,520 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:15:20,520 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:20,520 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46117 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 9 2024-12-05T22:15:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46117 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:15:20,521 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:15:20,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741837_1013 (size=395) 2024-12-05T22:15:20,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741837_1013 (size=395) 2024-12-05T22:15:20,531 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8de36a8b75009d40f2f4357f5b60a924, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba 2024-12-05T22:15:20,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37065 is added to blk_1073741838_1014 (size=78) 2024-12-05T22:15:20,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42555 is added to blk_1073741838_1014 (size=78) 2024-12-05T22:15:20,539 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:15:20,540 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1681): Closing 8de36a8b75009d40f2f4357f5b60a924, disabling compactions & flushes 2024-12-05T22:15:20,540 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:15:20,540 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:15:20,540 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. after waiting 0 ms 2024-12-05T22:15:20,540 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:15:20,540 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:15:20,540 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8de36a8b75009d40f2f4357f5b60a924: 2024-12-05T22:15:20,541 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:15:20,542 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733436920541"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733436920541"}]},"ts":"1733436920541"} 2024-12-05T22:15:20,543 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:15:20,545 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:15:20,545 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436920545"}]},"ts":"1733436920545"} 2024-12-05T22:15:20,547 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-12-05T22:15:20,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8de36a8b75009d40f2f4357f5b60a924, ASSIGN}] 2024-12-05T22:15:20,553 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8de36a8b75009d40f2f4357f5b60a924, ASSIGN 2024-12-05T22:15:20,554 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8de36a8b75009d40f2f4357f5b60a924, ASSIGN; state=OFFLINE, location=d029b80fd32e,38731,1733436919181; forceNewPlan=false, retain=false 2024-12-05T22:15:20,704 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=8de36a8b75009d40f2f4357f5b60a924, regionState=OPENING, regionLocation=d029b80fd32e,38731,1733436919181 2024-12-05T22:15:20,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 8de36a8b75009d40f2f4357f5b60a924, server=d029b80fd32e,38731,1733436919181}] 2024-12-05T22:15:20,859 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d029b80fd32e,38731,1733436919181 2024-12-05T22:15:20,865 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:15:20,865 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 8de36a8b75009d40f2f4357f5b60a924, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:15:20,865 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 8de36a8b75009d40f2f4357f5b60a924 2024-12-05T22:15:20,865 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:15:20,866 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 8de36a8b75009d40f2f4357f5b60a924 2024-12-05T22:15:20,866 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 8de36a8b75009d40f2f4357f5b60a924 2024-12-05T22:15:20,867 INFO [StoreOpener-8de36a8b75009d40f2f4357f5b60a924-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8de36a8b75009d40f2f4357f5b60a924 2024-12-05T22:15:20,869 INFO [StoreOpener-8de36a8b75009d40f2f4357f5b60a924-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8de36a8b75009d40f2f4357f5b60a924 columnFamilyName info 2024-12-05T22:15:20,869 DEBUG [StoreOpener-8de36a8b75009d40f2f4357f5b60a924-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:15:20,870 INFO [StoreOpener-8de36a8b75009d40f2f4357f5b60a924-1 {}] regionserver.HStore(327): Store=8de36a8b75009d40f2f4357f5b60a924/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:15:20,871 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/default/TestLogRolling-testLogRollOnPipelineRestart/8de36a8b75009d40f2f4357f5b60a924 2024-12-05T22:15:20,872 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/default/TestLogRolling-testLogRollOnPipelineRestart/8de36a8b75009d40f2f4357f5b60a924 2024-12-05T22:15:20,877 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 8de36a8b75009d40f2f4357f5b60a924 2024-12-05T22:15:20,879 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/default/TestLogRolling-testLogRollOnPipelineRestart/8de36a8b75009d40f2f4357f5b60a924/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:15:20,880 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 8de36a8b75009d40f2f4357f5b60a924; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=748800, jitterRate=-0.047851741313934326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:15:20,880 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 8de36a8b75009d40f2f4357f5b60a924: 2024-12-05T22:15:20,889 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924., pid=11, masterSystemTime=1733436920859 2024-12-05T22:15:20,898 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:15:20,898 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:15:20,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=8de36a8b75009d40f2f4357f5b60a924, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,38731,1733436919181 2024-12-05T22:15:20,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-05T22:15:20,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 8de36a8b75009d40f2f4357f5b60a924, server=d029b80fd32e,38731,1733436919181 in 225 msec 2024-12-05T22:15:20,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T22:15:20,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8de36a8b75009d40f2f4357f5b60a924, ASSIGN in 396 msec 2024-12-05T22:15:20,960 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:15:20,960 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436920960"}]},"ts":"1733436920960"} 2024-12-05T22:15:20,965 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-12-05T22:15:20,976 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:15:20,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 458 msec 2024-12-05T22:15:21,420 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:22,421 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:22,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-05T22:15:22,541 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-12-05T22:15:23,422 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:24,422 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:25,423 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:25,441 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:15:25,464 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:25,464 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:25,465 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:25,465 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:25,466 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:25,466 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:25,471 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:25,475 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:15:25,497 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-12-05T22:15:26,424 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:27,425 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:28,425 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:29,426 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:30,427 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46117 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:15:30,523 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart, procId: 9 completed 2024-12-05T22:15:30,526 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-12-05T22:15:30,526 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:15:31,427 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:32,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:32,533 INFO [Time-limited test {}] wal.TestLogRolling(337): log.getCurrentFileName()): hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 2024-12-05T22:15:32,534 WARN [ResponseProcessor for block BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:42555,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:32,534 WARN [DataStreamer for file /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123/d029b80fd32e%2C46117%2C1733436919123.1733436919268 block BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK], DatanodeInfoWithStorage[127.0.0.1:42555,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42555,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]) is bad. 2024-12-05T22:15:32,534 WARN [ResponseProcessor for block BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:42555,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:32,534 WARN [ResponseProcessor for block BP-1259869556-172.17.0.2-1733436918234:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1259869556-172.17.0.2-1733436918234:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:32,535 WARN [DataStreamer for file /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 block BP-1259869556-172.17.0.2-1733436918234:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1259869556-172.17.0.2-1733436918234:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42555,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK], DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42555,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]) is bad. 2024-12-05T22:15:32,535 WARN [DataStreamer for file /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.meta.1733436919933.meta block BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK], DatanodeInfoWithStorage[127.0.0.1:42555,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42555,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]) is bad. 2024-12-05T22:15:32,535 WARN [PacketResponder: BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42555] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,536 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_173845152_22 at /127.0.0.1:38540 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42555:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38540 dst: /127.0.0.1:42555 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,536 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_173845152_22 at /127.0.0.1:40478 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37065:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40478 dst: /127.0.0.1:37065 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,536 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_173845152_22 at /127.0.0.1:38542 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42555:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38542 dst: /127.0.0.1:42555 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,534 WARN [PacketResponder: BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42555] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,537 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_173845152_22 at /127.0.0.1:40470 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37065:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40470 dst: /127.0.0.1:37065 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,537 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-436770151_22 at /127.0.0.1:40436 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37065:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40436 dst: /127.0.0.1:37065 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,537 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-436770151_22 at /127.0.0.1:38488 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42555:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38488 dst: /127.0.0.1:42555 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,542 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bfb26c3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:32,543 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5f202bff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:15:32,543 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:15:32,543 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6058ad7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:15:32,543 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d286f08{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,STOPPED} 2024-12-05T22:15:32,545 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:15:32,545 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1259869556-172.17.0.2-1733436918234 (Datanode Uuid 0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1) service to localhost/127.0.0.1:38283 2024-12-05T22:15:32,546 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data3/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:32,546 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data4/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:32,546 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:15:32,546 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:15:32,546 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:15:32,563 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:15:32,568 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:15:32,571 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:15:32,571 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:15:32,572 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:15:32,572 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ebd01a4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:15:32,573 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4e597220{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:15:32,730 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26c3bfcf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/java.io.tmpdir/jetty-localhost-34297-hadoop-hdfs-3_4_1-tests_jar-_-any-4735663292590279027/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:32,730 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1965f5a7{HTTP/1.1, (http/1.1)}{localhost:34297} 2024-12-05T22:15:32,730 INFO [Time-limited test {}] server.Server(415): Started @186448ms 2024-12-05T22:15:32,732 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:15:32,765 WARN [ResponseProcessor for block BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:32,765 WARN [ResponseProcessor for block BP-1259869556-172.17.0.2-1733436918234:blk_1073741833_1017 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1259869556-172.17.0.2-1733436918234:blk_1073741833_1017 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:32,765 WARN [ResponseProcessor for block BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:32,766 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-436770151_22 at /127.0.0.1:33232 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37065:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33232 dst: /127.0.0.1:37065 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,767 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_173845152_22 at /127.0.0.1:33240 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37065:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33240 dst: /127.0.0.1:37065 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,767 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_173845152_22 at /127.0.0.1:33256 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37065:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33256 dst: /127.0.0.1:37065 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:32,776 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6997e634{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:32,777 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3479543b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:15:32,777 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:15:32,777 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3dddc2f1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:15:32,777 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3436fcc3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,STOPPED} 2024-12-05T22:15:32,784 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:15:32,784 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1259869556-172.17.0.2-1733436918234 (Datanode Uuid 4eb9d08a-9fc2-411d-9fb2-656977000172) service to localhost/127.0.0.1:38283 2024-12-05T22:15:32,786 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data2/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:32,786 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data1/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:32,786 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:15:32,786 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:15:32,786 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:15:32,798 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:15:32,802 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:15:32,806 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:15:32,806 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:15:32,806 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:15:32,808 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b368375{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:15:32,808 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1164a0b5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:15:32,857 WARN [Thread-1092 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:15:32,860 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x73e579b55ff6b8e8 with lease ID 0x7b4142fca855b644: from storage DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d node DatanodeRegistration(127.0.0.1:36551, datanodeUuid=0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1, infoPort=34993, infoSecurePort=0, ipcPort=38471, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T22:15:32,860 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x73e579b55ff6b8e8 with lease ID 0x7b4142fca855b644: from storage DS-a53518c1-f2e4-427a-93fb-a64e353ce496 node DatanodeRegistration(127.0.0.1:36551, datanodeUuid=0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1, infoPort=34993, infoSecurePort=0, ipcPort=38471, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:32,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@69b079e1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/java.io.tmpdir/jetty-localhost-42817-hadoop-hdfs-3_4_1-tests_jar-_-any-5278778557592191588/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:32,975 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5aa9c50{HTTP/1.1, (http/1.1)}{localhost:42817} 2024-12-05T22:15:32,976 INFO [Time-limited test {}] server.Server(415): Started @186693ms 2024-12-05T22:15:32,978 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:15:33,100 WARN [Thread-1123 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:15:33,110 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6d019c931ff5f5da with lease ID 0x7b4142fca855b645: from storage DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7 node DatanodeRegistration(127.0.0.1:38853, datanodeUuid=4eb9d08a-9fc2-411d-9fb2-656977000172, infoPort=40169, infoSecurePort=0, ipcPort=40463, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:33,110 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6d019c931ff5f5da with lease ID 0x7b4142fca855b645: from storage DS-8611fc5e-3e31-4abc-a2bf-ce8e07538000 node DatanodeRegistration(127.0.0.1:38853, datanodeUuid=4eb9d08a-9fc2-411d-9fb2-656977000172, infoPort=40169, infoSecurePort=0, ipcPort=40463, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:33,429 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:34,006 INFO [Time-limited test {}] wal.TestLogRolling(349): Data Nodes restarted 2024-12-05T22:15:34,009 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-12-05T22:15:34,010 WARN [RS:0;d029b80fd32e:38731.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=5, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:34,010 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C38731%2C1733436919181:(num 1733436919568) roll requested 2024-12-05T22:15:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38731 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:34,011 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38731%2C1733436919181.1733436934010 2024-12-05T22:15:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38731 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60872 deadline: 1733436944009, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL 2024-12-05T22:15:34,019 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 newFile=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 2024-12-05T22:15:34,019 WARN [regionserver/d029b80fd32e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL 2024-12-05T22:15:34,019 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 with entries=5, filesize=2.09 KB; new WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 2024-12-05T22:15:34,020 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34993:34993),(127.0.0.1/127.0.0.1:40169:40169)] 2024-12-05T22:15:34,020 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 is not closed yet, will try archiving it next time 2024-12-05T22:15:34,020 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:34,020 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:34,020 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 2024-12-05T22:15:34,020 WARN [IPC Server handler 0 on default port 38283 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1017 2024-12-05T22:15:34,021 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 after 1ms 2024-12-05T22:15:34,429 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:35,430 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:36,431 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:36,861 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1017: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-05T22:15:37,432 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:38,022 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 after 4002ms 2024-12-05T22:15:38,433 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:39,433 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:40,434 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:41,435 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:42,435 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:43,436 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:44,437 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:45,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:46,117 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-12-05T22:15:46,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:47,439 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:48,119 WARN [ResponseProcessor for block BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1018 java.io.IOException: Bad response ERROR for BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1018 from datanode DatanodeInfoWithStorage[127.0.0.1:38853,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:48,120 WARN [DataStreamer for file /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 block BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36551,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK], DatanodeInfoWithStorage[127.0.0.1:38853,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38853,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]) is bad. 2024-12-05T22:15:48,120 WARN [PacketResponder: BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1018, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38853] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:48,121 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_173845152_22 at /127.0.0.1:33760 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:36551:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33760 dst: /127.0.0.1:36551 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:48,121 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_173845152_22 at /127.0.0.1:53272 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:38853:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53272 dst: /127.0.0.1:38853 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:48,122 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@69b079e1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:48,122 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5aa9c50{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:15:48,122 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:15:48,123 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1164a0b5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:15:48,123 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b368375{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,STOPPED} 2024-12-05T22:15:48,127 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:15:48,127 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:15:48,127 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1259869556-172.17.0.2-1733436918234 (Datanode Uuid 4eb9d08a-9fc2-411d-9fb2-656977000172) service to localhost/127.0.0.1:38283 2024-12-05T22:15:48,127 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:15:48,127 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data1/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:48,127 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data2/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:48,128 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:15:48,136 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:15:48,140 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:15:48,140 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:15:48,140 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:15:48,140 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:15:48,141 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@77213bb6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:15:48,141 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56f9fb45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:15:48,258 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30ab6650{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/java.io.tmpdir/jetty-localhost-40901-hadoop-hdfs-3_4_1-tests_jar-_-any-16668278631266679199/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:48,258 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@382d9a44{HTTP/1.1, (http/1.1)}{localhost:40901} 2024-12-05T22:15:48,258 INFO [Time-limited test {}] server.Server(415): Started @201976ms 2024-12-05T22:15:48,260 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:15:48,280 WARN [ResponseProcessor for block BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1020 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:48,280 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_173845152_22 at /127.0.0.1:51472 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:36551:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51472 dst: /127.0.0.1:36551 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:15:48,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26c3bfcf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:48,282 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1965f5a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:15:48,282 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:15:48,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4e597220{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:15:48,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ebd01a4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,STOPPED} 2024-12-05T22:15:48,284 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:15:48,284 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:15:48,284 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1259869556-172.17.0.2-1733436918234 (Datanode Uuid 0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1) service to localhost/127.0.0.1:38283 2024-12-05T22:15:48,284 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:15:48,284 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data3/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:48,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data4/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:15:48,285 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:15:48,294 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:15:48,297 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:15:48,298 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:15:48,298 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:15:48,298 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:15:48,298 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7426272d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:15:48,299 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e9534fd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:15:48,371 WARN [Thread-1167 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:15:48,374 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5346055729cbf553 with lease ID 0x7b4142fca855b646: from storage DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7 node DatanodeRegistration(127.0.0.1:35973, datanodeUuid=4eb9d08a-9fc2-411d-9fb2-656977000172, infoPort=36625, infoSecurePort=0, ipcPort=36961, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:48,374 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5346055729cbf553 with lease ID 0x7b4142fca855b646: from storage DS-8611fc5e-3e31-4abc-a2bf-ce8e07538000 node DatanodeRegistration(127.0.0.1:35973, datanodeUuid=4eb9d08a-9fc2-411d-9fb2-656977000172, infoPort=36625, infoSecurePort=0, ipcPort=36961, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:48,426 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@489a548a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/java.io.tmpdir/jetty-localhost-42851-hadoop-hdfs-3_4_1-tests_jar-_-any-1116616233230354253/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:15:48,426 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@9bdcb4f{HTTP/1.1, (http/1.1)}{localhost:42851} 2024-12-05T22:15:48,426 INFO [Time-limited test {}] server.Server(415): Started @202144ms 2024-12-05T22:15:48,427 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:15:48,440 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:48,533 WARN [Thread-1198 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:15:48,536 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc659521c32fe8ef with lease ID 0x7b4142fca855b647: from storage DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d node DatanodeRegistration(127.0.0.1:39293, datanodeUuid=0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1, infoPort=34583, infoSecurePort=0, ipcPort=40555, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:48,537 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc659521c32fe8ef with lease ID 0x7b4142fca855b647: from storage DS-a53518c1-f2e4-427a-93fb-a64e353ce496 node DatanodeRegistration(127.0.0.1:39293, datanodeUuid=0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1, infoPort=34583, infoSecurePort=0, ipcPort=40555, storageInfo=lv=-57;cid=testClusterID;nsid=1671157902;c=1733436918234), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:15:49,103 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:15:49,332 WARN [master/d029b80fd32e:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=95, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:49,332 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C46117%2C1733436919123:(num 1733436919268) roll requested 2024-12-05T22:15:49,333 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:49,333 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C46117%2C1733436919123.1733436949332 2024-12-05T22:15:49,333 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:49,339 WARN [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL 2024-12-05T22:15:49,339 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123/d029b80fd32e%2C46117%2C1733436919123.1733436919268 with entries=92, filesize=45.98 KB; new WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123/d029b80fd32e%2C46117%2C1733436919123.1733436949332 2024-12-05T22:15:49,340 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34583:34583),(127.0.0.1/127.0.0.1:36625:36625)] 2024-12-05T22:15:49,340 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123/d029b80fd32e%2C46117%2C1733436919123.1733436919268 is not closed yet, will try archiving it next time 2024-12-05T22:15:49,340 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:49,341 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:49,341 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123/d029b80fd32e%2C46117%2C1733436919123.1733436919268 2024-12-05T22:15:49,341 WARN [IPC Server handler 2 on default port 38283 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123/d029b80fd32e%2C46117%2C1733436919123.1733436919268 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741830_1015 2024-12-05T22:15:49,341 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123/d029b80fd32e%2C46117%2C1733436919123.1733436919268 after 0ms 2024-12-05T22:15:49,440 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:49,446 INFO [Time-limited test {}] wal.TestLogRolling(366): Data Nodes restarted 2024-12-05T22:15:49,448 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-12-05T22:15:49,449 WARN [RS:0;d029b80fd32e:38731.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=8, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36551,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:49,449 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C38731%2C1733436919181:(num 1733436934010) roll requested 2024-12-05T22:15:49,450 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38731%2C1733436919181.1733436949449 2024-12-05T22:15:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38731 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36551,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38731 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60872 deadline: 1733436959448, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL 2024-12-05T22:15:49,456 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 newFile=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436949449 2024-12-05T22:15:49,457 WARN [regionserver/d029b80fd32e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL 2024-12-05T22:15:49,457 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436949449 2024-12-05T22:15:49,457 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34583:34583),(127.0.0.1/127.0.0.1:36625:36625)] 2024-12-05T22:15:49,457 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 is not closed yet, will try archiving it next time 2024-12-05T22:15:49,457 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36551,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:49,457 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36551,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:15:49,457 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 2024-12-05T22:15:49,458 WARN [IPC Server handler 4 on default port 38283 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741839_1020 2024-12-05T22:15:49,459 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 after 2ms 2024-12-05T22:15:50,411 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:15:50,413 INFO [RS-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50138, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:15:50,441 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:51,442 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:52,376 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741839_1020: GenerationStamp not matched, existing replica is blk_1073741839_1018 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-05T22:15:52,442 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:53,342 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123/d029b80fd32e%2C46117%2C1733436919123.1733436919268 after 4001ms 2024-12-05T22:15:53,443 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:53,459 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 after 4002ms 2024-12-05T22:15:54,444 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:54,536 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 2) replica(s): 0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-05T22:15:55,445 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:56,445 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:57,446 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:58,447 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:15:59,448 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:00,448 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:01,302 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T22:16:01,302 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T22:16:01,449 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:01,470 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38731%2C1733436919181.1733436961469 2024-12-05T22:16:01,477 DEBUG [Time-limited test {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436949449 newFile=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 2024-12-05T22:16:01,478 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436949449 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 2024-12-05T22:16:01,479 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36625:36625),(127.0.0.1/127.0.0.1:34583:34583)] 2024-12-05T22:16:01,479 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436949449 is not closed yet, will try archiving it next time 2024-12-05T22:16:01,479 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 2024-12-05T22:16:01,479 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 2024-12-05T22:16:01,479 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 after 0ms 2024-12-05T22:16:01,479 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 2024-12-05T22:16:01,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741841_1023 (size=1264) 2024-12-05T22:16:01,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741841_1023 (size=1264) 2024-12-05T22:16:01,486 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733436920330/Put/vlen=162/seqid=0] 2024-12-05T22:16:01,486 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [default/info:d/1733436920394/Put/vlen=9/seqid=0] 2024-12-05T22:16:01,486 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #5: [hbase/info:d/1733436920415/Put/vlen=7/seqid=0] 2024-12-05T22:16:01,486 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733436920880/Put/vlen=218/seqid=0] 2024-12-05T22:16:01,486 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [row1002/info:/1733436930530/Put/vlen=1045/seqid=0] 2024-12-05T22:16:01,486 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436919568 2024-12-05T22:16:01,486 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 2024-12-05T22:16:01,486 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 2024-12-05T22:16:01,487 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 after 1ms 2024-12-05T22:16:01,487 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 2024-12-05T22:16:01,490 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #6: [row1003/info:/1733436944114/Put/vlen=1045/seqid=0] 2024-12-05T22:16:01,490 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #7: [row1004/info:/1733436946118/Put/vlen=1045/seqid=0] 2024-12-05T22:16:01,490 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436934010 2024-12-05T22:16:01,490 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436949449 2024-12-05T22:16:01,490 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436949449 2024-12-05T22:16:01,490 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436949449 after 0ms 2024-12-05T22:16:01,491 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436949449 2024-12-05T22:16:01,494 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #9: [row1005/info:/1733436959468/Put/vlen=1045/seqid=0] 2024-12-05T22:16:01,494 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 2024-12-05T22:16:01,494 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 2024-12-05T22:16:01,494 WARN [IPC Server handler 0 on default port 38283 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741842_1025 2024-12-05T22:16:01,494 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 after 0ms 2024-12-05T22:16:02,376 WARN [ResponseProcessor for block BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:02,376 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-436770151_22 at /127.0.0.1:58390 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:35973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58390 dst: /127.0.0.1:35973 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35973 remote=/127.0.0.1:58390]. Total timeout mills is 60000, 59100 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:16:02,376 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-436770151_22 at /127.0.0.1:58984 [Receiving block BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:39293:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58984 dst: /127.0.0.1:39293 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:16:02,377 WARN [DataStreamer for file /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 block BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35973,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK], DatanodeInfoWithStorage[127.0.0.1:39293,DS-4b679f0c-8e9c-4964-9a5f-75ce873b1a1d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35973,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]) is bad. 2024-12-05T22:16:02,380 WARN [DataStreamer for file /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 block BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:02,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741842_1026 (size=85) 2024-12-05T22:16:02,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741842_1026 (size=85) 2024-12-05T22:16:02,450 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:03,451 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:04,451 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:05,452 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:05,495 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 after 4001ms 2024-12-05T22:16:05,495 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 2024-12-05T22:16:05,500 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 2024-12-05T22:16:05,501 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 8de36a8b75009d40f2f4357f5b60a924 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-12-05T22:16:05,501 WARN [RS:0;d029b80fd32e:38731.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=11, requesting roll of WAL org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,502 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C38731%2C1733436919181:(num 1733436961469) roll requested 2024-12-05T22:16:05,502 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 8de36a8b75009d40f2f4357f5b60a924: 2024-12-05T22:16:05,502 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38731%2C1733436919181.1733436965502 2024-12-05T22:16:05,502 INFO [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,502 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 9e2824609f96a6001d8013bb5c6d5a69 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-05T22:16:05,503 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 9e2824609f96a6001d8013bb5c6d5a69: 2024-12-05T22:16:05,503 INFO [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,503 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.42 KB 2024-12-05T22:16:05,503 WARN [RS_OPEN_META-regionserver/d029b80fd32e:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,504 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-12-05T22:16:05,504 INFO [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,507 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-05T22:16:05,507 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-05T22:16:05,507 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x545a7404 to 127.0.0.1:51580 2024-12-05T22:16:05,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:16:05,507 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T22:16:05,507 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1808595295, stopped=false 2024-12-05T22:16:05,507 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=d029b80fd32e,46117,1733436919123 2024-12-05T22:16:05,508 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 newFile=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436965502 2024-12-05T22:16:05,508 WARN [regionserver/d029b80fd32e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL 2024-12-05T22:16:05,508 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436965502 2024-12-05T22:16:05,509 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36625:36625),(127.0.0.1/127.0.0.1:34583:34583)] 2024-12-05T22:16:05,509 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 is not closed yet, will try archiving it next time 2024-12-05T22:16:05,509 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog d029b80fd32e%2C38731%2C1733436919181.meta:.meta(num 1733436919933) roll requested 2024-12-05T22:16:05,509 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,509 INFO [regionserver/d029b80fd32e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38731%2C1733436919181.meta.1733436965509.meta 2024-12-05T22:16:05,509 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1259869556-172.17.0.2-1733436918234:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,509 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 2024-12-05T22:16:05,510 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 after 1ms 2024-12-05T22:16:05,510 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.1733436961469 to hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/oldWALs/d029b80fd32e%2C38731%2C1733436919181.1733436961469 2024-12-05T22:16:05,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:16:05,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:05,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:16:05,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:05,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-05T22:16:05,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:16:05,512 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,38731,1733436919181' ***** 2024-12-05T22:16:05,512 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-05T22:16:05,512 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T22:16:05,512 INFO [RS:0;d029b80fd32e:38731 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T22:16:05,512 INFO [RS:0;d029b80fd32e:38731 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T22:16:05,512 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:16:05,512 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(3579): Received CLOSE for 8de36a8b75009d40f2f4357f5b60a924 2024-12-05T22:16:05,513 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-05T22:16:05,516 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:16:05,516 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(3579): Received CLOSE for 9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:16:05,516 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,38731,1733436919181 2024-12-05T22:16:05,516 DEBUG [RS:0;d029b80fd32e:38731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:16:05,516 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8de36a8b75009d40f2f4357f5b60a924, disabling compactions & flushes 2024-12-05T22:16:05,517 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T22:16:05,517 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T22:16:05,517 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T22:16:05,517 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:16:05,517 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:16:05,517 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-05T22:16:05,517 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. after waiting 0 ms 2024-12-05T22:16:05,517 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:16:05,517 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8de36a8b75009d40f2f4357f5b60a924 1/1 column families, dataSize=4.20 KB heapSize=4.98 KB 2024-12-05T22:16:05,517 WARN [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt? 2024-12-05T22:16:05,520 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-05T22:16:05,520 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1603): Online Regions={8de36a8b75009d40f2f4357f5b60a924=TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924., 9e2824609f96a6001d8013bb5c6d5a69=hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69., 1588230740=hbase:meta,,1.1588230740} 2024-12-05T22:16:05,520 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:16:05,520 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:16:05,520 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 8de36a8b75009d40f2f4357f5b60a924, 9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:16:05,520 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:16:05,520 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:16:05,520 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:16:05,521 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.89 KB 2024-12-05T22:16:05,521 WARN [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt? 2024-12-05T22:16:05,521 WARN [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt? 2024-12-05T22:16:05,521 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:16:05,521 ERROR [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2808): ***** ABORTING region server d029b80fd32e,38731,1733436919181: Unrecoverable exception while closing hbase:meta,,1.1588230740 ***** org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?] at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?] at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,521 ERROR [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2815): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-05T22:16:05,521 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-05T22:16:05,522 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-05T22:16:05,522 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-05T22:16:05,522 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-05T22:16:05,522 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2819): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 639410176 }, "NonHeapMemoryUsage": { "committed": 168755200, "init": 7667712, "max": -1, "used": 166935824 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-12-05T22:16:05,523 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46117 {}] master.MasterRpcServices(626): d029b80fd32e,38731,1733436919181 reported a fatal error: ***** ABORTING region server d029b80fd32e,38731,1733436919181: Unrecoverable exception while closing hbase:meta,,1.1588230740 ***** Cause: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) 2024-12-05T22:16:05,527 WARN [regionserver/d029b80fd32e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL 2024-12-05T22:16:05,527 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.meta.1733436919933.meta with entries=11, filesize=3.66 KB; new WAL /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.meta.1733436965509.meta 2024-12-05T22:16:05,527 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36625:36625),(127.0.0.1/127.0.0.1:34583:34583)] 2024-12-05T22:16:05,527 DEBUG [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.meta.1733436919933.meta is not closed yet, will try archiving it next time 2024-12-05T22:16:05,527 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,527 WARN [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37065,DS-843828ff-f5b6-4a6a-959d-f6052c65f8f7,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-05T22:16:05,528 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.meta.1733436919933.meta 2024-12-05T22:16:05,528 WARN [IPC Server handler 1 on default port 38283 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.meta.1733436919933.meta has not been closed. Lease recovery is in progress. RecoveryId = 1029 for block blk_1073741834_1016 2024-12-05T22:16:05,528 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.meta.1733436919933.meta after 0ms 2024-12-05T22:16:05,540 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/default/TestLogRolling-testLogRollOnPipelineRestart/8de36a8b75009d40f2f4357f5b60a924/.tmp/info/96c768ae51ba4686ad8a44dd6cf2652f is 1080, key is row1002/info:/1733436930530/Put/seqid=0 2024-12-05T22:16:05,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741845_1030 (size=9270) 2024-12-05T22:16:05,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741845_1030 (size=9270) 2024-12-05T22:16:05,545 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/default/TestLogRolling-testLogRollOnPipelineRestart/8de36a8b75009d40f2f4357f5b60a924/.tmp/info/96c768ae51ba4686ad8a44dd6cf2652f 2024-12-05T22:16:05,552 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/default/TestLogRolling-testLogRollOnPipelineRestart/8de36a8b75009d40f2f4357f5b60a924/.tmp/info/96c768ae51ba4686ad8a44dd6cf2652f as hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/default/TestLogRolling-testLogRollOnPipelineRestart/8de36a8b75009d40f2f4357f5b60a924/info/96c768ae51ba4686ad8a44dd6cf2652f 2024-12-05T22:16:05,558 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/default/TestLogRolling-testLogRollOnPipelineRestart/8de36a8b75009d40f2f4357f5b60a924/info/96c768ae51ba4686ad8a44dd6cf2652f, entries=4, sequenceid=12, filesize=9.1 K 2024-12-05T22:16:05,559 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 8de36a8b75009d40f2f4357f5b60a924 in 42ms, sequenceid=12, compaction requested=false 2024-12-05T22:16:05,563 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/data/default/TestLogRolling-testLogRollOnPipelineRestart/8de36a8b75009d40f2f4357f5b60a924/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1 2024-12-05T22:16:05,563 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:16:05,563 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8de36a8b75009d40f2f4357f5b60a924: 2024-12-05T22:16:05,563 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733436920517.8de36a8b75009d40f2f4357f5b60a924. 2024-12-05T22:16:05,564 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9e2824609f96a6001d8013bb5c6d5a69, disabling compactions & flushes 2024-12-05T22:16:05,564 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:16:05,564 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:16:05,564 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. after waiting 0 ms 2024-12-05T22:16:05,564 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:16:05,564 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9e2824609f96a6001d8013bb5c6d5a69: 2024-12-05T22:16:05,564 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:16:05,721 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(3579): Received CLOSE for 9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:16:05,721 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 9e2824609f96a6001d8013bb5c6d5a69, disabling compactions & flushes 2024-12-05T22:16:05,721 DEBUG [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 9e2824609f96a6001d8013bb5c6d5a69 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:16:05,721 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:16:05,721 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. after waiting 0 ms 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 9e2824609f96a6001d8013bb5c6d5a69: 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733436919976.9e2824609f96a6001d8013bb5c6d5a69. 2024-12-05T22:16:05,721 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:meta,,1.1588230740 2024-12-05T22:16:05,921 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1624): We were exiting though online regions are not empty, because some regions failed closing 2024-12-05T22:16:05,921 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,38731,1733436919181; all regions closed. 2024-12-05T22:16:05,922 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181 2024-12-05T22:16:05,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741844_1028 (size=93) 2024-12-05T22:16:05,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741844_1028 (size=93) 2024-12-05T22:16:06,453 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:06,458 INFO [regionserver/d029b80fd32e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T22:16:06,458 INFO [regionserver/d029b80fd32e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T22:16:06,539 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1016: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-05T22:16:07,432 INFO [regionserver/d029b80fd32e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:16:07,454 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:08,454 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:09,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:09,529 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181/d029b80fd32e%2C38731%2C1733436919181.meta.1733436919933.meta after 4001ms 2024-12-05T22:16:09,530 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/WALs/d029b80fd32e,38731,1733436919181 2024-12-05T22:16:09,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741843_1027 (size=1162) 2024-12-05T22:16:09,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741843_1027 (size=1162) 2024-12-05T22:16:09,533 DEBUG [RS:0;d029b80fd32e:38731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:16:09,533 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:16:09,533 INFO [RS:0;d029b80fd32e:38731 {}] hbase.ChoreService(370): Chore service for: regionserver/d029b80fd32e:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T22:16:09,533 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:16:09,533 INFO [RS:0;d029b80fd32e:38731 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38731 2024-12-05T22:16:09,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d029b80fd32e,38731,1733436919181 2024-12-05T22:16:09,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:16:09,541 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007f3188912b28@79c58f85 rejected from java.util.concurrent.ThreadPoolExecutor@7dfa0465[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 15] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T22:16:09,542 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d029b80fd32e,38731,1733436919181] 2024-12-05T22:16:09,542 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing d029b80fd32e,38731,1733436919181; numProcessing=1 2024-12-05T22:16:09,543 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/d029b80fd32e,38731,1733436919181 already deleted, retry=false 2024-12-05T22:16:09,543 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; d029b80fd32e,38731,1733436919181 expired; onlineServers=0 2024-12-05T22:16:09,543 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,46117,1733436919123' ***** 2024-12-05T22:16:09,543 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T22:16:09,544 DEBUG [M:0;d029b80fd32e:46117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9576914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:16:09,544 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,46117,1733436919123 2024-12-05T22:16:09,544 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,46117,1733436919123; all regions closed. 2024-12-05T22:16:09,544 DEBUG [M:0;d029b80fd32e:46117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:16:09,544 DEBUG [M:0;d029b80fd32e:46117 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T22:16:09,544 DEBUG [M:0;d029b80fd32e:46117 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T22:16:09,544 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T22:16:09,544 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436919333 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436919333,5,FailOnTimeoutGroup] 2024-12-05T22:16:09,544 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436919333 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436919333,5,FailOnTimeoutGroup] 2024-12-05T22:16:09,544 INFO [M:0;d029b80fd32e:46117 {}] hbase.ChoreService(370): Chore service for: master/d029b80fd32e:0 had [] on shutdown 2024-12-05T22:16:09,544 DEBUG [M:0;d029b80fd32e:46117 {}] master.HMaster(1733): Stopping service threads 2024-12-05T22:16:09,544 INFO [M:0;d029b80fd32e:46117 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T22:16:09,545 INFO [M:0;d029b80fd32e:46117 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T22:16:09,545 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T22:16:09,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T22:16:09,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:09,545 DEBUG [M:0;d029b80fd32e:46117 {}] zookeeper.ZKUtil(347): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T22:16:09,545 WARN [M:0;d029b80fd32e:46117 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T22:16:09,545 INFO [M:0;d029b80fd32e:46117 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-05T22:16:09,545 INFO [M:0;d029b80fd32e:46117 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T22:16:09,546 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:16:09,546 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:16:09,546 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:16:09,546 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:16:09,546 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:16:09,546 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:16:09,546 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.08 KB heapSize=49.23 KB 2024-12-05T22:16:09,562 DEBUG [M:0;d029b80fd32e:46117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d0421f99977c4c8d97f8f4e72fcd1898 is 82, key is hbase:meta,,1/info:regioninfo/1733436919953/Put/seqid=0 2024-12-05T22:16:09,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741846_1031 (size=5672) 2024-12-05T22:16:09,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741846_1031 (size=5672) 2024-12-05T22:16:09,568 INFO [M:0;d029b80fd32e:46117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d0421f99977c4c8d97f8f4e72fcd1898 2024-12-05T22:16:09,590 DEBUG [M:0;d029b80fd32e:46117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8afa43a80be3431bb201961d9d0ca4ee is 777, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733436920977/Put/seqid=0 2024-12-05T22:16:09,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741847_1032 (size=7468) 2024-12-05T22:16:09,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741847_1032 (size=7468) 2024-12-05T22:16:09,596 INFO [M:0;d029b80fd32e:46117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.48 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8afa43a80be3431bb201961d9d0ca4ee 2024-12-05T22:16:09,615 DEBUG [M:0;d029b80fd32e:46117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/002dbb753f0e4472ac1ef1ccc1642249 is 69, key is d029b80fd32e,38731,1733436919181/rs:state/1733436919415/Put/seqid=0 2024-12-05T22:16:09,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741848_1033 (size=5156) 2024-12-05T22:16:09,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741848_1033 (size=5156) 2024-12-05T22:16:09,621 INFO [M:0;d029b80fd32e:46117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/002dbb753f0e4472ac1ef1ccc1642249 2024-12-05T22:16:09,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:16:09,642 INFO [RS:0;d029b80fd32e:38731 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,38731,1733436919181; zookeeper connection closed. 2024-12-05T22:16:09,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38731-0x10184b237d20001, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:16:09,642 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@361ce049 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@361ce049 2024-12-05T22:16:09,643 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T22:16:09,644 DEBUG [M:0;d029b80fd32e:46117 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d5d90d8502a14f6db32689746eaed4e1 is 52, key is load_balancer_on/state:d/1733436920510/Put/seqid=0 2024-12-05T22:16:09,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741849_1034 (size=5056) 2024-12-05T22:16:09,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741849_1034 (size=5056) 2024-12-05T22:16:09,650 INFO [M:0;d029b80fd32e:46117 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d5d90d8502a14f6db32689746eaed4e1 2024-12-05T22:16:09,657 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d0421f99977c4c8d97f8f4e72fcd1898 as hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d0421f99977c4c8d97f8f4e72fcd1898 2024-12-05T22:16:09,662 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d0421f99977c4c8d97f8f4e72fcd1898, entries=8, sequenceid=96, filesize=5.5 K 2024-12-05T22:16:09,663 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8afa43a80be3431bb201961d9d0ca4ee as hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8afa43a80be3431bb201961d9d0ca4ee 2024-12-05T22:16:09,669 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8afa43a80be3431bb201961d9d0ca4ee, entries=11, sequenceid=96, filesize=7.3 K 2024-12-05T22:16:09,672 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/002dbb753f0e4472ac1ef1ccc1642249 as hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/002dbb753f0e4472ac1ef1ccc1642249 2024-12-05T22:16:09,677 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/002dbb753f0e4472ac1ef1ccc1642249, entries=1, sequenceid=96, filesize=5.0 K 2024-12-05T22:16:09,678 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d5d90d8502a14f6db32689746eaed4e1 as hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d5d90d8502a14f6db32689746eaed4e1 2024-12-05T22:16:09,684 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:38283/user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d5d90d8502a14f6db32689746eaed4e1, entries=1, sequenceid=96, filesize=4.9 K 2024-12-05T22:16:09,689 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.08 KB/41040, heapSize ~49.16 KB/50344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 143ms, sequenceid=96, compaction requested=false 2024-12-05T22:16:09,693 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:16:09,693 DEBUG [M:0;d029b80fd32e:46117 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:16:09,695 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/f57d8f36-83d7-fccc-d9b6-50f4d47794ba/MasterData/WALs/d029b80fd32e,46117,1733436919123 2024-12-05T22:16:09,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35973 is added to blk_1073741840_1021 (size=757) 2024-12-05T22:16:09,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39293 is added to blk_1073741840_1021 (size=757) 2024-12-05T22:16:10,103 INFO [M:0;d029b80fd32e:46117 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-05T22:16:10,103 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:16:10,103 INFO [M:0;d029b80fd32e:46117 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:46117 2024-12-05T22:16:10,106 DEBUG [M:0;d029b80fd32e:46117 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/d029b80fd32e,46117,1733436919123 already deleted, retry=false 2024-12-05T22:16:10,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:16:10,208 INFO [M:0;d029b80fd32e:46117 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,46117,1733436919123; zookeeper connection closed. 2024-12-05T22:16:10,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46117-0x10184b237d20000, quorum=127.0.0.1:51580, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:16:10,211 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@489a548a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:16:10,211 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@9bdcb4f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:16:10,211 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:16:10,211 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e9534fd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:16:10,211 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7426272d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,STOPPED} 2024-12-05T22:16:10,213 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:16:10,213 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:16:10,213 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:16:10,213 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1259869556-172.17.0.2-1733436918234 (Datanode Uuid 0034abe3-7cf5-4e8e-acbc-55ca3e4bfeb1) service to localhost/127.0.0.1:38283 2024-12-05T22:16:10,214 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data3/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:16:10,214 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data4/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:16:10,214 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:16:10,216 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30ab6650{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:16:10,217 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@382d9a44{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:16:10,217 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:16:10,217 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56f9fb45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:16:10,217 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@77213bb6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,STOPPED} 2024-12-05T22:16:10,219 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:16:10,219 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:16:10,219 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:16:10,219 WARN [BP-1259869556-172.17.0.2-1733436918234 heartbeating to localhost/127.0.0.1:38283 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1259869556-172.17.0.2-1733436918234 (Datanode Uuid 4eb9d08a-9fc2-411d-9fb2-656977000172) service to localhost/127.0.0.1:38283 2024-12-05T22:16:10,219 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data1/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:16:10,220 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/cluster_a76d6517-b91f-453a-c5db-20fe106e9a20/dfs/data/data2/current/BP-1259869556-172.17.0.2-1733436918234 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:16:10,220 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:16:10,226 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@393bcdc0{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:16:10,227 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15fa4d7a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:16:10,227 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:16:10,227 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47516fb1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:16:10,227 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3338083e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir/,STOPPED} 2024-12-05T22:16:10,236 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-05T22:16:10,255 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-05T22:16:10,262 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=100 (was 86) Potentially hanging thread: nioEventLoopGroup-28-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38283 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-28-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:38283 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-27-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-29-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-28-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-26-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-29-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:38283 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:38283 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-27-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-26-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38283 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38283 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-29-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38283 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:38283 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-26-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-27-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=444 (was 424) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 303) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2914 (was 4302) 2024-12-05T22:16:10,268 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=100, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=2914 2024-12-05T22:16:10,268 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.log.dir so I do NOT create it in target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/bb97e904-e891-6984-3920-ba9e61cb4afa/hadoop.tmp.dir so I do NOT create it in target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998, deleteOnExit=true 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/test.cache.data in system properties and HBase conf 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.log.dir in system properties and HBase conf 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-05T22:16:10,269 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:16:10,269 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/nfs.dump.dir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/java.io.tmpdir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T22:16:10,270 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T22:16:10,284 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:16:10,367 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:16:10,371 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:16:10,372 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:16:10,372 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:16:10,372 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:16:10,373 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:16:10,373 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20b89acb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:16:10,374 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f70eed0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:16:10,456 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:10,497 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@903456d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/java.io.tmpdir/jetty-localhost-41499-hadoop-hdfs-3_4_1-tests_jar-_-any-8599042148786641505/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:16:10,498 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@58429b8e{HTTP/1.1, (http/1.1)}{localhost:41499} 2024-12-05T22:16:10,498 INFO [Time-limited test {}] server.Server(415): Started @224216ms 2024-12-05T22:16:10,511 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:16:10,576 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,576 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,576 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,576 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,577 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,577 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,580 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,580 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,580 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,581 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,586 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,587 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,587 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,587 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,609 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:16:10,611 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,611 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,611 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,611 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,625 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,626 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,626 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,626 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,627 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,627 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,632 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,632 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,632 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,635 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:10,645 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:16:10,648 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:16:10,649 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:16:10,649 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:16:10,650 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:16:10,650 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d858d58{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:16:10,651 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18c4ea87{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:16:10,777 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a0d04be{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/java.io.tmpdir/jetty-localhost-38911-hadoop-hdfs-3_4_1-tests_jar-_-any-456658747272106818/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:16:10,778 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@634fdd28{HTTP/1.1, (http/1.1)}{localhost:38911} 2024-12-05T22:16:10,778 INFO [Time-limited test {}] server.Server(415): Started @224496ms 2024-12-05T22:16:10,780 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:16:10,828 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:16:10,832 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:16:10,833 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:16:10,833 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:16:10,833 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:16:10,834 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29d5bebf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:16:10,834 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@214f83d4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:16:10,889 WARN [Thread-1369 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/dfs/data/data1/current/BP-909378404-172.17.0.2-1733436970301/current, will proceed with Du for space computation calculation, 2024-12-05T22:16:10,889 WARN [Thread-1370 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/dfs/data/data2/current/BP-909378404-172.17.0.2-1733436970301/current, will proceed with Du for space computation calculation, 2024-12-05T22:16:10,910 WARN [Thread-1348 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:16:10,913 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x869297c04b68a8bd with lease ID 0x9386fcbdb873a605: Processing first storage report for DS-1b996dee-882a-42a4-a4a3-0b439d44e53f from datanode DatanodeRegistration(127.0.0.1:40371, datanodeUuid=9b840b89-7497-4486-8b0e-8b9fa804c0b2, infoPort=35049, infoSecurePort=0, ipcPort=43253, storageInfo=lv=-57;cid=testClusterID;nsid=1601872436;c=1733436970301) 2024-12-05T22:16:10,913 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x869297c04b68a8bd with lease ID 0x9386fcbdb873a605: from storage DS-1b996dee-882a-42a4-a4a3-0b439d44e53f node DatanodeRegistration(127.0.0.1:40371, datanodeUuid=9b840b89-7497-4486-8b0e-8b9fa804c0b2, infoPort=35049, infoSecurePort=0, ipcPort=43253, storageInfo=lv=-57;cid=testClusterID;nsid=1601872436;c=1733436970301), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T22:16:10,913 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x869297c04b68a8bd with lease ID 0x9386fcbdb873a605: Processing first storage report for DS-67821e84-8355-4d34-b7e6-6df5caddc2c9 from datanode DatanodeRegistration(127.0.0.1:40371, datanodeUuid=9b840b89-7497-4486-8b0e-8b9fa804c0b2, infoPort=35049, infoSecurePort=0, ipcPort=43253, storageInfo=lv=-57;cid=testClusterID;nsid=1601872436;c=1733436970301) 2024-12-05T22:16:10,913 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x869297c04b68a8bd with lease ID 0x9386fcbdb873a605: from storage DS-67821e84-8355-4d34-b7e6-6df5caddc2c9 node DatanodeRegistration(127.0.0.1:40371, datanodeUuid=9b840b89-7497-4486-8b0e-8b9fa804c0b2, infoPort=35049, infoSecurePort=0, ipcPort=43253, storageInfo=lv=-57;cid=testClusterID;nsid=1601872436;c=1733436970301), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:16:10,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b6aeddc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/java.io.tmpdir/jetty-localhost-39177-hadoop-hdfs-3_4_1-tests_jar-_-any-2326163404230791952/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:16:10,955 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@607e1d6{HTTP/1.1, (http/1.1)}{localhost:39177} 2024-12-05T22:16:10,955 INFO [Time-limited test {}] server.Server(415): Started @224672ms 2024-12-05T22:16:10,956 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:16:11,056 WARN [Thread-1395 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/dfs/data/data3/current/BP-909378404-172.17.0.2-1733436970301/current, will proceed with Du for space computation calculation, 2024-12-05T22:16:11,056 WARN [Thread-1396 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/dfs/data/data4/current/BP-909378404-172.17.0.2-1733436970301/current, will proceed with Du for space computation calculation, 2024-12-05T22:16:11,073 WARN [Thread-1384 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:16:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5f691aa171df8e66 with lease ID 0x9386fcbdb873a606: Processing first storage report for DS-7563d7be-3c0b-4002-ab6c-42d5bbcc9050 from datanode DatanodeRegistration(127.0.0.1:39331, datanodeUuid=b4d5cc32-ac0f-47d9-a915-357b0155f981, infoPort=37593, infoSecurePort=0, ipcPort=37277, storageInfo=lv=-57;cid=testClusterID;nsid=1601872436;c=1733436970301) 2024-12-05T22:16:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f691aa171df8e66 with lease ID 0x9386fcbdb873a606: from storage DS-7563d7be-3c0b-4002-ab6c-42d5bbcc9050 node DatanodeRegistration(127.0.0.1:39331, datanodeUuid=b4d5cc32-ac0f-47d9-a915-357b0155f981, infoPort=37593, infoSecurePort=0, ipcPort=37277, storageInfo=lv=-57;cid=testClusterID;nsid=1601872436;c=1733436970301), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:16:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5f691aa171df8e66 with lease ID 0x9386fcbdb873a606: Processing first storage report for DS-6816d2d8-a89d-4e4f-bdf2-136baabc3342 from datanode DatanodeRegistration(127.0.0.1:39331, datanodeUuid=b4d5cc32-ac0f-47d9-a915-357b0155f981, infoPort=37593, infoSecurePort=0, ipcPort=37277, storageInfo=lv=-57;cid=testClusterID;nsid=1601872436;c=1733436970301) 2024-12-05T22:16:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f691aa171df8e66 with lease ID 0x9386fcbdb873a606: from storage DS-6816d2d8-a89d-4e4f-bdf2-136baabc3342 node DatanodeRegistration(127.0.0.1:39331, datanodeUuid=b4d5cc32-ac0f-47d9-a915-357b0155f981, infoPort=37593, infoSecurePort=0, ipcPort=37277, storageInfo=lv=-57;cid=testClusterID;nsid=1601872436;c=1733436970301), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:16:11,083 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957 2024-12-05T22:16:11,085 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/zookeeper_0, clientPort=50103, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T22:16:11,086 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=50103 2024-12-05T22:16:11,087 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:16:11,089 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:16:11,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:16:11,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:16:11,100 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3 with version=8 2024-12-05T22:16:11,100 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/hbase-staging 2024-12-05T22:16:11,102 INFO [Time-limited test {}] client.ConnectionUtils(129): master/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:16:11,102 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:16:11,102 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:16:11,102 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:16:11,102 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:16:11,102 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:16:11,102 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:16:11,102 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:16:11,103 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:43289 2024-12-05T22:16:11,103 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:16:11,104 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:16:11,107 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:43289 connecting to ZooKeeper ensemble=127.0.0.1:50103 2024-12-05T22:16:11,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432890x0, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:16:11,113 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43289-0x10184b302dc0000 connected 2024-12-05T22:16:11,136 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:16:11,137 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:16:11,137 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:16:11,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43289 2024-12-05T22:16:11,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43289 2024-12-05T22:16:11,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43289 2024-12-05T22:16:11,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43289 2024-12-05T22:16:11,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43289 2024-12-05T22:16:11,144 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3, hbase.cluster.distributed=false 2024-12-05T22:16:11,161 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:16:11,161 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:16:11,161 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:16:11,161 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:16:11,161 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:16:11,161 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:16:11,161 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:16:11,161 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:16:11,162 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:33147 2024-12-05T22:16:11,162 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T22:16:11,165 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T22:16:11,165 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:16:11,167 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:16:11,169 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:33147 connecting to ZooKeeper ensemble=127.0.0.1:50103 2024-12-05T22:16:11,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:331470x0, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:16:11,172 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33147-0x10184b302dc0001 connected 2024-12-05T22:16:11,172 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:16:11,173 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:16:11,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:16:11,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33147 2024-12-05T22:16:11,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33147 2024-12-05T22:16:11,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33147 2024-12-05T22:16:11,177 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33147 2024-12-05T22:16:11,177 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33147 2024-12-05T22:16:11,177 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/d029b80fd32e,43289,1733436971101 2024-12-05T22:16:11,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:16:11,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:16:11,181 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d029b80fd32e,43289,1733436971101 2024-12-05T22:16:11,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:16:11,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:16:11,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,183 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:16:11,183 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d029b80fd32e,43289,1733436971101 from backup master directory 2024-12-05T22:16:11,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:16:11,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d029b80fd32e,43289,1733436971101 2024-12-05T22:16:11,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:16:11,184 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:16:11,184 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:16:11,184 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d029b80fd32e,43289,1733436971101 2024-12-05T22:16:11,189 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d029b80fd32e:43289 2024-12-05T22:16:11,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:16:11,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:16:11,195 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/hbase.id with ID: 8c0fe8b8-2d76-4914-9128-49c93aa68e4d 2024-12-05T22:16:11,205 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:16:11,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:16:11,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:16:11,216 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:16:11,217 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T22:16:11,217 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:16:11,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:16:11,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:16:11,225 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store 2024-12-05T22:16:11,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:16:11,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:16:11,232 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:16:11,232 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:16:11,232 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:16:11,232 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:16:11,232 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:16:11,232 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:16:11,232 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:16:11,232 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:16:11,233 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/.initializing 2024-12-05T22:16:11,233 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/WALs/d029b80fd32e,43289,1733436971101 2024-12-05T22:16:11,236 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C43289%2C1733436971101, suffix=, logDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/WALs/d029b80fd32e,43289,1733436971101, archiveDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/oldWALs, maxLogs=10 2024-12-05T22:16:11,236 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C43289%2C1733436971101.1733436971236 2024-12-05T22:16:11,241 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/WALs/d029b80fd32e,43289,1733436971101/d029b80fd32e%2C43289%2C1733436971101.1733436971236 2024-12-05T22:16:11,241 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37593:37593),(127.0.0.1/127.0.0.1:35049:35049)] 2024-12-05T22:16:11,241 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:16:11,241 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:16:11,241 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:16:11,241 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:16:11,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:16:11,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T22:16:11,244 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:11,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:16:11,245 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:16:11,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T22:16:11,246 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:11,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:16:11,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:16:11,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T22:16:11,247 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:11,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:16:11,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:16:11,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T22:16:11,249 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:11,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:16:11,250 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:16:11,250 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:16:11,253 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T22:16:11,254 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:16:11,256 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:16:11,256 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=785097, jitterRate=-0.0016982704401016235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T22:16:11,257 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:16:11,257 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T22:16:11,261 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4681b704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:16:11,262 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-05T22:16:11,262 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T22:16:11,262 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T22:16:11,262 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T22:16:11,262 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T22:16:11,263 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-05T22:16:11,263 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T22:16:11,265 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T22:16:11,265 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T22:16:11,268 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-05T22:16:11,269 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T22:16:11,269 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T22:16:11,270 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-05T22:16:11,271 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T22:16:11,271 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T22:16:11,272 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-05T22:16:11,273 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T22:16:11,274 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T22:16:11,276 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T22:16:11,278 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T22:16:11,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:16:11,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:16:11,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,281 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=d029b80fd32e,43289,1733436971101, sessionid=0x10184b302dc0000, setting cluster-up flag (Was=false) 2024-12-05T22:16:11,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,293 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T22:16:11,293 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,43289,1733436971101 2024-12-05T22:16:11,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,312 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T22:16:11,313 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,43289,1733436971101 2024-12-05T22:16:11,316 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-05T22:16:11,316 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-05T22:16:11,316 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T22:16:11,316 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d029b80fd32e,43289,1733436971101 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T22:16:11,317 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:16:11,317 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:16:11,317 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:16:11,317 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:16:11,317 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d029b80fd32e:0, corePoolSize=10, maxPoolSize=10 2024-12-05T22:16:11,317 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,317 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:16:11,317 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,318 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733437001318 2024-12-05T22:16:11,318 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T22:16:11,318 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T22:16:11,318 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T22:16:11,318 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T22:16:11,318 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T22:16:11,318 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T22:16:11,318 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,318 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:16:11,319 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-05T22:16:11,319 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T22:16:11,319 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T22:16:11,319 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T22:16:11,319 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T22:16:11,319 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T22:16:11,319 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436971319,5,FailOnTimeoutGroup] 2024-12-05T22:16:11,320 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436971320,5,FailOnTimeoutGroup] 2024-12-05T22:16:11,320 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,320 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:11,320 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T22:16:11,320 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,320 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,320 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:16:11,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:16:11,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:16:11,328 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-05T22:16:11,328 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3 2024-12-05T22:16:11,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:16:11,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:16:11,398 DEBUG [RS:0;d029b80fd32e:33147 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d029b80fd32e:33147 2024-12-05T22:16:11,402 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1008): ClusterId : 8c0fe8b8-2d76-4914-9128-49c93aa68e4d 2024-12-05T22:16:11,402 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T22:16:11,419 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T22:16:11,419 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T22:16:11,424 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T22:16:11,425 DEBUG [RS:0;d029b80fd32e:33147 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6917a2ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:16:11,425 DEBUG [RS:0;d029b80fd32e:33147 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d409e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:16:11,425 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-05T22:16:11,425 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-05T22:16:11,425 DEBUG [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-05T22:16:11,426 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(3073): reportForDuty to master=d029b80fd32e,43289,1733436971101 with isa=d029b80fd32e/172.17.0.2:33147, startcode=1733436971160 2024-12-05T22:16:11,426 DEBUG [RS:0;d029b80fd32e:33147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:16:11,429 INFO [RS-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59903, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:16:11,430 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43289 {}] master.ServerManager(332): Checking decommissioned status of RegionServer d029b80fd32e,33147,1733436971160 2024-12-05T22:16:11,430 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43289 {}] master.ServerManager(486): Registering regionserver=d029b80fd32e,33147,1733436971160 2024-12-05T22:16:11,431 DEBUG [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3 2024-12-05T22:16:11,431 DEBUG [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:36979 2024-12-05T22:16:11,431 DEBUG [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-05T22:16:11,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:16:11,433 DEBUG [RS:0;d029b80fd32e:33147 {}] zookeeper.ZKUtil(111): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d029b80fd32e,33147,1733436971160 2024-12-05T22:16:11,433 WARN [RS:0;d029b80fd32e:33147 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:16:11,433 INFO [RS:0;d029b80fd32e:33147 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:16:11,433 DEBUG [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160 2024-12-05T22:16:11,434 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d029b80fd32e,33147,1733436971160] 2024-12-05T22:16:11,436 DEBUG [RS:0;d029b80fd32e:33147 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-05T22:16:11,437 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T22:16:11,438 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T22:16:11,439 INFO [RS:0;d029b80fd32e:33147 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T22:16:11,439 INFO [RS:0;d029b80fd32e:33147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,439 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-05T22:16:11,440 INFO [RS:0;d029b80fd32e:33147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:16:11,440 DEBUG [RS:0;d029b80fd32e:33147 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:16:11,441 INFO [RS:0;d029b80fd32e:33147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,441 INFO [RS:0;d029b80fd32e:33147 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,441 INFO [RS:0;d029b80fd32e:33147 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,441 INFO [RS:0;d029b80fd32e:33147 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,441 INFO [RS:0;d029b80fd32e:33147 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,33147,1733436971160-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:16:11,455 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T22:16:11,455 INFO [RS:0;d029b80fd32e:33147 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,33147,1733436971160-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:11,456 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:11,471 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.Replication(204): d029b80fd32e,33147,1733436971160 started 2024-12-05T22:16:11,471 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1767): Serving as d029b80fd32e,33147,1733436971160, RpcServer on d029b80fd32e/172.17.0.2:33147, sessionid=0x10184b302dc0001 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d029b80fd32e,33147,1733436971160 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,33147,1733436971160' 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d029b80fd32e,33147,1733436971160 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,33147,1733436971160' 2024-12-05T22:16:11,471 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T22:16:11,472 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T22:16:11,472 DEBUG [RS:0;d029b80fd32e:33147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T22:16:11,472 INFO [RS:0;d029b80fd32e:33147 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T22:16:11,472 INFO [RS:0;d029b80fd32e:33147 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T22:16:11,574 INFO [RS:0;d029b80fd32e:33147 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C33147%2C1733436971160, suffix=, logDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160, archiveDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/oldWALs, maxLogs=32 2024-12-05T22:16:11,575 INFO [RS:0;d029b80fd32e:33147 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C33147%2C1733436971160.1733436971575 2024-12-05T22:16:11,581 INFO [RS:0;d029b80fd32e:33147 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733436971575 2024-12-05T22:16:11,581 DEBUG [RS:0;d029b80fd32e:33147 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35049:35049),(127.0.0.1/127.0.0.1:37593:37593)] 2024-12-05T22:16:11,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:16:11,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:16:11,741 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:16:11,741 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:11,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:16:11,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:16:11,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:16:11,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:11,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:16:11,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:16:11,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:16:11,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:11,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:16:11,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740 2024-12-05T22:16:11,747 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740 2024-12-05T22:16:11,749 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:16:11,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:16:11,753 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:16:11,753 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713908, jitterRate=-0.0922200083732605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:16:11,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:16:11,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:16:11,753 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:16:11,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:16:11,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:16:11,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:16:11,754 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:16:11,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:16:11,755 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:16:11,755 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-05T22:16:11,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T22:16:11,756 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T22:16:11,757 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T22:16:11,908 DEBUG [d029b80fd32e:43289 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T22:16:11,908 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d029b80fd32e,33147,1733436971160 2024-12-05T22:16:11,909 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,33147,1733436971160, state=OPENING 2024-12-05T22:16:11,911 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T22:16:11,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:11,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:16:11,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:16:11,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=d029b80fd32e,33147,1733436971160}] 2024-12-05T22:16:12,065 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33147,1733436971160 2024-12-05T22:16:12,066 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T22:16:12,068 INFO [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54950, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T22:16:12,071 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-05T22:16:12,071 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:16:12,073 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C33147%2C1733436971160.meta, suffix=.meta, logDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160, archiveDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/oldWALs, maxLogs=32 2024-12-05T22:16:12,074 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C33147%2C1733436971160.meta.1733436972074.meta 2024-12-05T22:16:12,080 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.meta.1733436972074.meta 2024-12-05T22:16:12,080 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35049:35049),(127.0.0.1/127.0.0.1:37593:37593)] 2024-12-05T22:16:12,080 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:16:12,080 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T22:16:12,080 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T22:16:12,080 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T22:16:12,080 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T22:16:12,080 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:16:12,080 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-05T22:16:12,080 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-05T22:16:12,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:16:12,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:16:12,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:12,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:16:12,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:16:12,084 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:16:12,084 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:12,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:16:12,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:16:12,086 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:16:12,086 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:12,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:16:12,087 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740 2024-12-05T22:16:12,088 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740 2024-12-05T22:16:12,090 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:16:12,091 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:16:12,092 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=696979, jitterRate=-0.11374589800834656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:16:12,092 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:16:12,093 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733436972065 2024-12-05T22:16:12,095 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T22:16:12,095 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-05T22:16:12,096 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,33147,1733436971160 2024-12-05T22:16:12,096 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,33147,1733436971160, state=OPEN 2024-12-05T22:16:12,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:16:12,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:16:12,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:16:12,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:16:12,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T22:16:12,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=d029b80fd32e,33147,1733436971160 in 192 msec 2024-12-05T22:16:12,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T22:16:12,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 352 msec 2024-12-05T22:16:12,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 796 msec 2024-12-05T22:16:12,111 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733436972111, completionTime=-1 2024-12-05T22:16:12,111 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T22:16:12,111 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-05T22:16:12,112 DEBUG [hconnection-0x4bafb8fb-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:16:12,113 INFO [RS-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54966, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:16:12,114 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-05T22:16:12,114 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733437032114 2024-12-05T22:16:12,114 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733437092114 2024-12-05T22:16:12,114 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec 2024-12-05T22:16:12,120 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,43289,1733436971101-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:12,120 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,43289,1733436971101-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:12,120 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,43289,1733436971101-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:12,120 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d029b80fd32e:43289, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:12,120 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:12,120 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-05T22:16:12,120 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:16:12,121 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-05T22:16:12,122 DEBUG [master/d029b80fd32e:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-05T22:16:12,122 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:16:12,123 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:12,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:16:12,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:16:12,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:16:12,133 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 5309833186e01b7e2225320581f62c6a, NAME => 'hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3 2024-12-05T22:16:12,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:16:12,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:16:12,143 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:16:12,143 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 5309833186e01b7e2225320581f62c6a, disabling compactions & flushes 2024-12-05T22:16:12,143 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:16:12,143 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:16:12,143 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. after waiting 0 ms 2024-12-05T22:16:12,143 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:16:12,143 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:16:12,143 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 5309833186e01b7e2225320581f62c6a: 2024-12-05T22:16:12,144 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:16:12,144 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733436972144"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733436972144"}]},"ts":"1733436972144"} 2024-12-05T22:16:12,146 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:16:12,147 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:16:12,147 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436972147"}]},"ts":"1733436972147"} 2024-12-05T22:16:12,149 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-05T22:16:12,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5309833186e01b7e2225320581f62c6a, ASSIGN}] 2024-12-05T22:16:12,154 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5309833186e01b7e2225320581f62c6a, ASSIGN 2024-12-05T22:16:12,155 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=5309833186e01b7e2225320581f62c6a, ASSIGN; state=OFFLINE, location=d029b80fd32e,33147,1733436971160; forceNewPlan=false, retain=false 2024-12-05T22:16:12,305 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5309833186e01b7e2225320581f62c6a, regionState=OPENING, regionLocation=d029b80fd32e,33147,1733436971160 2024-12-05T22:16:12,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 5309833186e01b7e2225320581f62c6a, server=d029b80fd32e,33147,1733436971160}] 2024-12-05T22:16:12,457 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:12,461 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33147,1733436971160 2024-12-05T22:16:12,466 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:16:12,466 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 5309833186e01b7e2225320581f62c6a, NAME => 'hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:16:12,466 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 5309833186e01b7e2225320581f62c6a 2024-12-05T22:16:12,466 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:16:12,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 5309833186e01b7e2225320581f62c6a 2024-12-05T22:16:12,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 5309833186e01b7e2225320581f62c6a 2024-12-05T22:16:12,471 INFO [StoreOpener-5309833186e01b7e2225320581f62c6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5309833186e01b7e2225320581f62c6a 2024-12-05T22:16:12,473 INFO [StoreOpener-5309833186e01b7e2225320581f62c6a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5309833186e01b7e2225320581f62c6a columnFamilyName info 2024-12-05T22:16:12,473 DEBUG [StoreOpener-5309833186e01b7e2225320581f62c6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:12,474 INFO [StoreOpener-5309833186e01b7e2225320581f62c6a-1 {}] regionserver.HStore(327): Store=5309833186e01b7e2225320581f62c6a/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:16:12,474 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/namespace/5309833186e01b7e2225320581f62c6a 2024-12-05T22:16:12,475 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/namespace/5309833186e01b7e2225320581f62c6a 2024-12-05T22:16:12,477 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 5309833186e01b7e2225320581f62c6a 2024-12-05T22:16:12,479 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/namespace/5309833186e01b7e2225320581f62c6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:16:12,480 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 5309833186e01b7e2225320581f62c6a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=757072, jitterRate=-0.0373338907957077}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:16:12,480 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 5309833186e01b7e2225320581f62c6a: 2024-12-05T22:16:12,481 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a., pid=6, masterSystemTime=1733436972461 2024-12-05T22:16:12,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:16:12,483 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:16:12,484 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5309833186e01b7e2225320581f62c6a, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,33147,1733436971160 2024-12-05T22:16:12,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T22:16:12,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 5309833186e01b7e2225320581f62c6a, server=d029b80fd32e,33147,1733436971160 in 181 msec 2024-12-05T22:16:12,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T22:16:12,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=5309833186e01b7e2225320581f62c6a, ASSIGN in 339 msec 2024-12-05T22:16:12,496 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:16:12,496 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436972496"}]},"ts":"1733436972496"} 2024-12-05T22:16:12,499 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-05T22:16:12,505 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:16:12,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 385 msec 2024-12-05T22:16:12,523 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-05T22:16:12,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:12,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:16:12,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:16:12,532 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-05T22:16:12,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-05T22:16:12,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:16:12,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 14 msec 2024-12-05T22:16:12,555 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-05T22:16:12,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:16:12,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 10 msec 2024-12-05T22:16:12,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-05T22:16:12,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-05T22:16:12,583 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.399sec 2024-12-05T22:16:12,583 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T22:16:12,583 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T22:16:12,583 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T22:16:12,583 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T22:16:12,583 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T22:16:12,583 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,43289,1733436971101-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:16:12,583 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,43289,1733436971101-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T22:16:12,585 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-05T22:16:12,585 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T22:16:12,585 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,43289,1733436971101-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:16:12,681 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c8c60d5 to 127.0.0.1:50103 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55c7ddc7 2024-12-05T22:16:12,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ea8370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:16:12,687 DEBUG [hconnection-0x4c7aaae3-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:16:12,689 INFO [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54970, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:16:12,691 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=d029b80fd32e,43289,1733436971101 2024-12-05T22:16:12,691 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:16:12,695 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-05T22:16:12,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T22:16:12,698 INFO [RS-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58028, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T22:16:12,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-05T22:16:12,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-05T22:16:12,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:16:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:16:12,702 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:16:12,702 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:12,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 9 2024-12-05T22:16:12,703 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:16:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:16:12,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741837_1013 (size=405) 2024-12-05T22:16:12,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741837_1013 (size=405) 2024-12-05T22:16:12,715 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1b82557da140bfa884096bff9a8d8674, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3 2024-12-05T22:16:12,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741838_1014 (size=88) 2024-12-05T22:16:12,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741838_1014 (size=88) 2024-12-05T22:16:12,727 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:16:12,727 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1681): Closing 1b82557da140bfa884096bff9a8d8674, disabling compactions & flushes 2024-12-05T22:16:12,728 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:12,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:12,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. after waiting 0 ms 2024-12-05T22:16:12,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:12,728 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:12,728 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1b82557da140bfa884096bff9a8d8674: 2024-12-05T22:16:12,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:16:12,730 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733436972729"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733436972729"}]},"ts":"1733436972729"} 2024-12-05T22:16:12,737 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:16:12,739 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:16:12,739 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436972739"}]},"ts":"1733436972739"} 2024-12-05T22:16:12,741 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-12-05T22:16:12,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1b82557da140bfa884096bff9a8d8674, ASSIGN}] 2024-12-05T22:16:12,748 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1b82557da140bfa884096bff9a8d8674, ASSIGN 2024-12-05T22:16:12,750 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1b82557da140bfa884096bff9a8d8674, ASSIGN; state=OFFLINE, location=d029b80fd32e,33147,1733436971160; forceNewPlan=false, retain=false 2024-12-05T22:16:12,900 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=1b82557da140bfa884096bff9a8d8674, regionState=OPENING, regionLocation=d029b80fd32e,33147,1733436971160 2024-12-05T22:16:12,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 1b82557da140bfa884096bff9a8d8674, server=d029b80fd32e,33147,1733436971160}] 2024-12-05T22:16:13,055 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33147,1733436971160 2024-12-05T22:16:13,060 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:13,060 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 1b82557da140bfa884096bff9a8d8674, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:16:13,061 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 1b82557da140bfa884096bff9a8d8674 2024-12-05T22:16:13,061 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:16:13,061 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 1b82557da140bfa884096bff9a8d8674 2024-12-05T22:16:13,061 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 1b82557da140bfa884096bff9a8d8674 2024-12-05T22:16:13,066 INFO [StoreOpener-1b82557da140bfa884096bff9a8d8674-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1b82557da140bfa884096bff9a8d8674 2024-12-05T22:16:13,072 INFO [StoreOpener-1b82557da140bfa884096bff9a8d8674-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b82557da140bfa884096bff9a8d8674 columnFamilyName info 2024-12-05T22:16:13,072 DEBUG [StoreOpener-1b82557da140bfa884096bff9a8d8674-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:16:13,080 INFO [StoreOpener-1b82557da140bfa884096bff9a8d8674-1 {}] regionserver.HStore(327): Store=1b82557da140bfa884096bff9a8d8674/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:16:13,081 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674 2024-12-05T22:16:13,082 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674 2024-12-05T22:16:13,085 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 1b82557da140bfa884096bff9a8d8674 2024-12-05T22:16:13,088 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:16:13,088 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 1b82557da140bfa884096bff9a8d8674; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=841910, jitterRate=0.07054464519023895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:16:13,089 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 1b82557da140bfa884096bff9a8d8674: 2024-12-05T22:16:13,090 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674., pid=11, masterSystemTime=1733436973055 2024-12-05T22:16:13,093 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:13,093 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:13,094 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=1b82557da140bfa884096bff9a8d8674, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,33147,1733436971160 2024-12-05T22:16:13,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-05T22:16:13,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 1b82557da140bfa884096bff9a8d8674, server=d029b80fd32e,33147,1733436971160 in 193 msec 2024-12-05T22:16:13,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T22:16:13,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=1b82557da140bfa884096bff9a8d8674, ASSIGN in 352 msec 2024-12-05T22:16:13,102 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:16:13,102 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733436973102"}]},"ts":"1733436973102"} 2024-12-05T22:16:13,104 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-12-05T22:16:13,107 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:16:13,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 408 msec 2024-12-05T22:16:13,457 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:14,458 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:15,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:16,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:17,460 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:17,583 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:16:17,585 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,585 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,585 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,585 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,600 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,600 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,600 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,601 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,601 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,601 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,604 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,604 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,604 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,606 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:16:17,613 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-12-05T22:16:18,461 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:19,461 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:20,462 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:20,463 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta after 68048ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor199.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:16:21,463 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:22,464 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:22,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:16:22,541 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-12-05T22:16:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:16:22,705 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 9 completed 2024-12-05T22:16:22,707 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:16:22,707 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:22,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush hbase:namespace 2024-12-05T22:16:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace 2024-12-05T22:16:22,723 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_PREPARE 2024-12-05T22:16:22,724 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T22:16:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-05T22:16:22,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T22:16:22,884 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33147,1733436971160 2024-12-05T22:16:22,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33147 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-12-05T22:16:22,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:16:22,886 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2837): Flushing 5309833186e01b7e2225320581f62c6a 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-05T22:16:22,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/namespace/5309833186e01b7e2225320581f62c6a/.tmp/info/1861366d8aa448c38dbe03b907103429 is 45, key is default/info:d/1733436972537/Put/seqid=0 2024-12-05T22:16:22,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741839_1015 (size=5037) 2024-12-05T22:16:22,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741839_1015 (size=5037) 2024-12-05T22:16:22,909 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/namespace/5309833186e01b7e2225320581f62c6a/.tmp/info/1861366d8aa448c38dbe03b907103429 2024-12-05T22:16:22,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/namespace/5309833186e01b7e2225320581f62c6a/.tmp/info/1861366d8aa448c38dbe03b907103429 as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/namespace/5309833186e01b7e2225320581f62c6a/info/1861366d8aa448c38dbe03b907103429 2024-12-05T22:16:22,922 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/namespace/5309833186e01b7e2225320581f62c6a/info/1861366d8aa448c38dbe03b907103429, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T22:16:22,923 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 5309833186e01b7e2225320581f62c6a in 37ms, sequenceid=6, compaction requested=false 2024-12-05T22:16:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2538): Flush status journal for 5309833186e01b7e2225320581f62c6a: 2024-12-05T22:16:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:16:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=13 2024-12-05T22:16:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster(4106): Remote procedure done, pid=13 2024-12-05T22:16:22,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-12-05T22:16:22,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 203 msec 2024-12-05T22:16:22,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace in 213 msec 2024-12-05T22:16:23,464 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:24,465 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:25,466 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:26,466 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:27,467 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:28,468 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:29,468 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:30,469 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:31,470 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:32,471 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-12-05T22:16:32,726 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: hbase:namespace, procId: 12 completed 2024-12-05T22:16:32,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:16:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:16:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-05T22:16:32,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-05T22:16:32,737 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T22:16:32,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T22:16:32,889 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33147,1733436971160 2024-12-05T22:16:32,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33147 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-12-05T22:16:32,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:32,890 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2837): Flushing 1b82557da140bfa884096bff9a8d8674 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-05T22:16:32,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/ce6ca7f5e9ed49dca73ff8e9d95f5414 is 1080, key is row0001/info:/1733436992730/Put/seqid=0 2024-12-05T22:16:32,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741840_1016 (size=6033) 2024-12-05T22:16:32,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741840_1016 (size=6033) 2024-12-05T22:16:32,913 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/ce6ca7f5e9ed49dca73ff8e9d95f5414 2024-12-05T22:16:32,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/ce6ca7f5e9ed49dca73ff8e9d95f5414 as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/ce6ca7f5e9ed49dca73ff8e9d95f5414 2024-12-05T22:16:32,925 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/ce6ca7f5e9ed49dca73ff8e9d95f5414, entries=1, sequenceid=5, filesize=5.9 K 2024-12-05T22:16:32,926 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1b82557da140bfa884096bff9a8d8674 in 36ms, sequenceid=5, compaction requested=false 2024-12-05T22:16:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2538): Flush status journal for 1b82557da140bfa884096bff9a8d8674: 2024-12-05T22:16:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-05T22:16:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster(4106): Remote procedure done, pid=15 2024-12-05T22:16:32,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14 2024-12-05T22:16:32,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 191 msec 2024-12-05T22:16:32,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 197 msec 2024-12-05T22:16:33,471 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:34,472 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:35,472 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:36,473 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:37,473 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:38,474 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:39,475 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:40,475 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:41,082 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:16:41,476 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:42,477 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-12-05T22:16:42,738 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 14 completed 2024-12-05T22:16:42,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:16:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] procedure2.ProcedureExecutor(1098): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:16:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-05T22:16:42,746 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-05T22:16:42,746 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T22:16:42,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T22:16:42,899 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33147,1733436971160 2024-12-05T22:16:42,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33147 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-12-05T22:16:42,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:42,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2837): Flushing 1b82557da140bfa884096bff9a8d8674 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-05T22:16:42,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/47ac423a67cb44b4bae654ee2854cf73 is 1080, key is row0002/info:/1733437002738/Put/seqid=0 2024-12-05T22:16:42,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741841_1017 (size=6033) 2024-12-05T22:16:42,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741841_1017 (size=6033) 2024-12-05T22:16:42,912 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/47ac423a67cb44b4bae654ee2854cf73 2024-12-05T22:16:42,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/47ac423a67cb44b4bae654ee2854cf73 as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/47ac423a67cb44b4bae654ee2854cf73 2024-12-05T22:16:42,925 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/47ac423a67cb44b4bae654ee2854cf73, entries=1, sequenceid=9, filesize=5.9 K 2024-12-05T22:16:42,926 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1b82557da140bfa884096bff9a8d8674 in 26ms, sequenceid=9, compaction requested=false 2024-12-05T22:16:42,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2538): Flush status journal for 1b82557da140bfa884096bff9a8d8674: 2024-12-05T22:16:42,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:42,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=17 2024-12-05T22:16:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster(4106): Remote procedure done, pid=17 2024-12-05T22:16:42,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-12-05T22:16:42,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 181 msec 2024-12-05T22:16:42,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 185 msec 2024-12-05T22:16:43,425 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:16:43,427 INFO [RS-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59656, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:16:43,477 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:44,478 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:45,479 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:46,479 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:47,480 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:48,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:49,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:50,482 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:51,482 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:52,483 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-12-05T22:16:52,747 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 16 completed 2024-12-05T22:16:52,749 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C33147%2C1733436971160.1733437012749 2024-12-05T22:16:52,757 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733436971575 with entries=13, filesize=6.41 KB; new WAL /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733437012749 2024-12-05T22:16:52,758 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35049:35049),(127.0.0.1/127.0.0.1:37593:37593)] 2024-12-05T22:16:52,758 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733436971575 is not closed yet, will try archiving it next time 2024-12-05T22:16:52,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741833_1009 (size=6574) 2024-12-05T22:16:52,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741833_1009 (size=6574) 2024-12-05T22:16:52,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:16:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:16:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-05T22:16:52,763 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-05T22:16:52,764 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T22:16:52,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T22:16:52,915 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33147,1733436971160 2024-12-05T22:16:52,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33147 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-12-05T22:16:52,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:52,916 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2837): Flushing 1b82557da140bfa884096bff9a8d8674 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-05T22:16:52,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/293ed400a74342be913620e51f9fbea8 is 1080, key is row0003/info:/1733437012748/Put/seqid=0 2024-12-05T22:16:52,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741843_1019 (size=6033) 2024-12-05T22:16:52,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741843_1019 (size=6033) 2024-12-05T22:16:52,930 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/293ed400a74342be913620e51f9fbea8 2024-12-05T22:16:52,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/293ed400a74342be913620e51f9fbea8 as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/293ed400a74342be913620e51f9fbea8 2024-12-05T22:16:52,944 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/293ed400a74342be913620e51f9fbea8, entries=1, sequenceid=13, filesize=5.9 K 2024-12-05T22:16:52,945 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1b82557da140bfa884096bff9a8d8674 in 29ms, sequenceid=13, compaction requested=true 2024-12-05T22:16:52,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2538): Flush status journal for 1b82557da140bfa884096bff9a8d8674: 2024-12-05T22:16:52,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:16:52,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-05T22:16:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster(4106): Remote procedure done, pid=19 2024-12-05T22:16:52,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-12-05T22:16:52,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 183 msec 2024-12-05T22:16:52,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 188 msec 2024-12-05T22:16:53,330 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T22:16:53,330 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T22:16:53,484 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:54,484 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:55,485 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:56,486 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:57,467 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 5309833186e01b7e2225320581f62c6a, had cached 0 bytes from a total of 5037 2024-12-05T22:16:57,486 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:58,061 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1b82557da140bfa884096bff9a8d8674, had cached 0 bytes from a total of 18099 2024-12-05T22:16:58,487 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:16:59,487 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:00,488 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:01,489 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:02,489 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-12-05T22:17:02,765 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 18 completed 2024-12-05T22:17:02,765 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:17:02,766 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:17:02,767 DEBUG [Time-limited test {}] regionserver.HStore(1540): 1b82557da140bfa884096bff9a8d8674/info is initiating minor compaction (all files) 2024-12-05T22:17:02,767 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T22:17:02,767 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:02,767 INFO [Time-limited test {}] regionserver.HRegion(2351): Starting compaction of 1b82557da140bfa884096bff9a8d8674/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:17:02,767 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/ce6ca7f5e9ed49dca73ff8e9d95f5414, hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/47ac423a67cb44b4bae654ee2854cf73, hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/293ed400a74342be913620e51f9fbea8] into tmpdir=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp, totalSize=17.7 K 2024-12-05T22:17:02,768 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting ce6ca7f5e9ed49dca73ff8e9d95f5414, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733436992730 2024-12-05T22:17:02,768 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 47ac423a67cb44b4bae654ee2854cf73, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733437002738 2024-12-05T22:17:02,769 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 293ed400a74342be913620e51f9fbea8, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733437012748 2024-12-05T22:17:02,781 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1b82557da140bfa884096bff9a8d8674#info#compaction#29 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:17:02,782 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/8aaeeb6fa7ff4fb394fb31a5607e7c1c is 1080, key is row0001/info:/1733436992730/Put/seqid=0 2024-12-05T22:17:02,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741844_1020 (size=8296) 2024-12-05T22:17:02,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741844_1020 (size=8296) 2024-12-05T22:17:02,794 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/8aaeeb6fa7ff4fb394fb31a5607e7c1c as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/8aaeeb6fa7ff4fb394fb31a5607e7c1c 2024-12-05T22:17:02,801 INFO [Time-limited test {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 1b82557da140bfa884096bff9a8d8674/info of 1b82557da140bfa884096bff9a8d8674 into 8aaeeb6fa7ff4fb394fb31a5607e7c1c(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:17:02,801 DEBUG [Time-limited test {}] regionserver.HRegion(2381): Compaction status journal for 1b82557da140bfa884096bff9a8d8674: 2024-12-05T22:17:02,803 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C33147%2C1733436971160.1733437022803 2024-12-05T22:17:02,811 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733437012749 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733437022803 2024-12-05T22:17:02,811 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37593:37593),(127.0.0.1/127.0.0.1:35049:35049)] 2024-12-05T22:17:02,811 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733437012749 is not closed yet, will try archiving it next time 2024-12-05T22:17:02,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741842_1018 (size=2520) 2024-12-05T22:17:02,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741842_1018 (size=2520) 2024-12-05T22:17:02,813 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733436971575 to hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/oldWALs/d029b80fd32e%2C33147%2C1733436971160.1733436971575 2024-12-05T22:17:02,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:17:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] procedure2.ProcedureExecutor(1098): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:17:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-05T22:17:02,818 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-05T22:17:02,819 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T22:17:02,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T22:17:02,971 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33147,1733436971160 2024-12-05T22:17:02,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33147 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-12-05T22:17:02,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:17:02,972 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2837): Flushing 1b82557da140bfa884096bff9a8d8674 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-05T22:17:02,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/d95378392e654c808741703101cdf88c is 1080, key is row0000/info:/1733437022802/Put/seqid=0 2024-12-05T22:17:02,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741846_1022 (size=6033) 2024-12-05T22:17:02,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741846_1022 (size=6033) 2024-12-05T22:17:02,983 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/d95378392e654c808741703101cdf88c 2024-12-05T22:17:02,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/d95378392e654c808741703101cdf88c as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/d95378392e654c808741703101cdf88c 2024-12-05T22:17:02,995 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/d95378392e654c808741703101cdf88c, entries=1, sequenceid=18, filesize=5.9 K 2024-12-05T22:17:02,996 INFO [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1b82557da140bfa884096bff9a8d8674 in 24ms, sequenceid=18, compaction requested=false 2024-12-05T22:17:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2538): Flush status journal for 1b82557da140bfa884096bff9a8d8674: 2024-12-05T22:17:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:17:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=21 2024-12-05T22:17:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.HMaster(4106): Remote procedure done, pid=21 2024-12-05T22:17:02,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=21, resume processing ppid=20 2024-12-05T22:17:02,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 178 msec 2024-12-05T22:17:03,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec 2024-12-05T22:17:03,490 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:04,491 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:05,491 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:06,492 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:07,493 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:08,493 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:09,494 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:10,495 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:11,082 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:17:11,495 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:12,125 DEBUG [master/d029b80fd32e:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T22:17:12,126 DEBUG [master/d029b80fd32e:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 5309833186e01b7e2225320581f62c6a changed from -1.0 to 0.0, refreshing cache 2024-12-05T22:17:12,496 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43289 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-12-05T22:17:12,820 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 20 completed 2024-12-05T22:17:12,822 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C33147%2C1733436971160.1733437032822 2024-12-05T22:17:12,836 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733437022803 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733437032822 2024-12-05T22:17:12,837 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35049:35049),(127.0.0.1/127.0.0.1:37593:37593)] 2024-12-05T22:17:12,837 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733437022803 is not closed yet, will try archiving it next time 2024-12-05T22:17:12,837 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-05T22:17:12,837 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-05T22:17:12,837 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0c8c60d5 to 127.0.0.1:50103 2024-12-05T22:17:12,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:17:12,837 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T22:17:12,837 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1941805239, stopped=false 2024-12-05T22:17:12,837 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=d029b80fd32e,43289,1733436971101 2024-12-05T22:17:12,838 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160/d029b80fd32e%2C33147%2C1733436971160.1733437012749 to hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/oldWALs/d029b80fd32e%2C33147%2C1733436971160.1733437012749 2024-12-05T22:17:12,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741845_1021 (size=2026) 2024-12-05T22:17:12,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741845_1021 (size=2026) 2024-12-05T22:17:12,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:17:12,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-05T22:17:12,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:12,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:17:12,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:17:12,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:12,849 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,33147,1733436971160' ***** 2024-12-05T22:17:12,849 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-05T22:17:12,849 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:17:12,853 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:17:12,861 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T22:17:12,861 INFO [RS:0;d029b80fd32e:33147 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T22:17:12,861 INFO [RS:0;d029b80fd32e:33147 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T22:17:12,861 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-05T22:17:12,861 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(3579): Received CLOSE for 1b82557da140bfa884096bff9a8d8674 2024-12-05T22:17:12,863 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(3579): Received CLOSE for 5309833186e01b7e2225320581f62c6a 2024-12-05T22:17:12,864 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,33147,1733436971160 2024-12-05T22:17:12,864 DEBUG [RS:0;d029b80fd32e:33147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:17:12,864 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T22:17:12,864 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T22:17:12,864 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T22:17:12,864 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-05T22:17:12,864 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close 2024-12-05T22:17:12,864 DEBUG [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1603): Online Regions={1b82557da140bfa884096bff9a8d8674=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674., 1588230740=hbase:meta,,1.1588230740, 5309833186e01b7e2225320581f62c6a=hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a.} 2024-12-05T22:17:12,864 DEBUG [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 1b82557da140bfa884096bff9a8d8674, 5309833186e01b7e2225320581f62c6a 2024-12-05T22:17:12,864 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1b82557da140bfa884096bff9a8d8674, disabling compactions & flushes 2024-12-05T22:17:12,864 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:17:12,865 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:17:12,865 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:17:12,865 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:17:12,865 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:17:12,865 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. after waiting 0 ms 2024-12-05T22:17:12,865 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:17:12,865 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:17:12,865 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:17:12,865 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1b82557da140bfa884096bff9a8d8674 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-05T22:17:12,865 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=3.05 KB heapSize=5.55 KB 2024-12-05T22:17:12,875 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/cbc92cf9db494592a1dd457249a04297 is 1080, key is row0001/info:/1733437032820/Put/seqid=0 2024-12-05T22:17:12,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741848_1024 (size=6033) 2024-12-05T22:17:12,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741848_1024 (size=6033) 2024-12-05T22:17:12,886 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/cbc92cf9db494592a1dd457249a04297 2024-12-05T22:17:12,891 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/.tmp/info/8071392a156c48198342df0f720cdd11 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674./info:regioninfo/1733436973093/Put/seqid=0 2024-12-05T22:17:12,899 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/.tmp/info/cbc92cf9db494592a1dd457249a04297 as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/cbc92cf9db494592a1dd457249a04297 2024-12-05T22:17:12,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741849_1025 (size=8430) 2024-12-05T22:17:12,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741849_1025 (size=8430) 2024-12-05T22:17:12,901 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.79 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/.tmp/info/8071392a156c48198342df0f720cdd11 2024-12-05T22:17:12,905 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/cbc92cf9db494592a1dd457249a04297, entries=1, sequenceid=22, filesize=5.9 K 2024-12-05T22:17:12,908 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 1b82557da140bfa884096bff9a8d8674 in 42ms, sequenceid=22, compaction requested=true 2024-12-05T22:17:12,912 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/ce6ca7f5e9ed49dca73ff8e9d95f5414, hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/47ac423a67cb44b4bae654ee2854cf73, hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/293ed400a74342be913620e51f9fbea8] to archive 2024-12-05T22:17:12,914 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T22:17:12,916 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/ce6ca7f5e9ed49dca73ff8e9d95f5414 to hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/ce6ca7f5e9ed49dca73ff8e9d95f5414 2024-12-05T22:17:12,918 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/47ac423a67cb44b4bae654ee2854cf73 to hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/47ac423a67cb44b4bae654ee2854cf73 2024-12-05T22:17:12,920 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/293ed400a74342be913620e51f9fbea8 to hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/info/293ed400a74342be913620e51f9fbea8 2024-12-05T22:17:12,931 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/.tmp/table/69692f6ad79c4bfeb2279e7aba0ee0e8 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733436973102/Put/seqid=0 2024-12-05T22:17:12,933 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/1b82557da140bfa884096bff9a8d8674/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-12-05T22:17:12,934 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:17:12,934 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1b82557da140bfa884096bff9a8d8674: 2024-12-05T22:17:12,934 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733436972699.1b82557da140bfa884096bff9a8d8674. 2024-12-05T22:17:12,934 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5309833186e01b7e2225320581f62c6a, disabling compactions & flushes 2024-12-05T22:17:12,934 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:17:12,934 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:17:12,934 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. after waiting 0 ms 2024-12-05T22:17:12,934 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:17:12,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741850_1026 (size=5532) 2024-12-05T22:17:12,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741850_1026 (size=5532) 2024-12-05T22:17:12,939 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=264 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/.tmp/table/69692f6ad79c4bfeb2279e7aba0ee0e8 2024-12-05T22:17:12,942 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/namespace/5309833186e01b7e2225320581f62c6a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T22:17:12,943 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:17:12,943 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5309833186e01b7e2225320581f62c6a: 2024-12-05T22:17:12,943 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733436972120.5309833186e01b7e2225320581f62c6a. 2024-12-05T22:17:12,948 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/.tmp/info/8071392a156c48198342df0f720cdd11 as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/info/8071392a156c48198342df0f720cdd11 2024-12-05T22:17:12,956 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/info/8071392a156c48198342df0f720cdd11, entries=20, sequenceid=14, filesize=8.2 K 2024-12-05T22:17:12,957 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/.tmp/table/69692f6ad79c4bfeb2279e7aba0ee0e8 as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/table/69692f6ad79c4bfeb2279e7aba0ee0e8 2024-12-05T22:17:12,963 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/table/69692f6ad79c4bfeb2279e7aba0ee0e8, entries=4, sequenceid=14, filesize=5.4 K 2024-12-05T22:17:12,964 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~3.05 KB/3122, heapSize ~5.27 KB/5400, currentSize=0 B/0 for 1588230740 in 99ms, sequenceid=14, compaction requested=false 2024-12-05T22:17:12,974 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1 2024-12-05T22:17:12,975 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T22:17:12,975 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:17:12,975 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:17:12,975 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T22:17:13,065 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,33147,1733436971160; all regions closed. 2024-12-05T22:17:13,065 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160 2024-12-05T22:17:13,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741834_1010 (size=4570) 2024-12-05T22:17:13,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741834_1010 (size=4570) 2024-12-05T22:17:13,069 DEBUG [RS:0;d029b80fd32e:33147 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/oldWALs 2024-12-05T22:17:13,069 INFO [RS:0;d029b80fd32e:33147 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog d029b80fd32e%2C33147%2C1733436971160.meta:.meta(num 1733436972074) 2024-12-05T22:17:13,070 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/WALs/d029b80fd32e,33147,1733436971160 2024-12-05T22:17:13,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741847_1023 (size=1545) 2024-12-05T22:17:13,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741847_1023 (size=1545) 2024-12-05T22:17:13,074 DEBUG [RS:0;d029b80fd32e:33147 {}] wal.AbstractFSWAL(1071): Moved 2 WAL file(s) to /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/oldWALs 2024-12-05T22:17:13,074 INFO [RS:0;d029b80fd32e:33147 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog d029b80fd32e%2C33147%2C1733436971160:(num 1733437032822) 2024-12-05T22:17:13,074 DEBUG [RS:0;d029b80fd32e:33147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:17:13,074 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:17:13,074 INFO [RS:0;d029b80fd32e:33147 {}] hbase.ChoreService(370): Chore service for: regionserver/d029b80fd32e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T22:17:13,074 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:17:13,075 INFO [RS:0;d029b80fd32e:33147 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:33147 2024-12-05T22:17:13,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:17:13,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d029b80fd32e,33147,1733436971160 2024-12-05T22:17:13,079 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d029b80fd32e,33147,1733436971160] 2024-12-05T22:17:13,079 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing d029b80fd32e,33147,1733436971160; numProcessing=1 2024-12-05T22:17:13,080 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/d029b80fd32e,33147,1733436971160 already deleted, retry=false 2024-12-05T22:17:13,080 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; d029b80fd32e,33147,1733436971160 expired; onlineServers=0 2024-12-05T22:17:13,080 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,43289,1733436971101' ***** 2024-12-05T22:17:13,080 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T22:17:13,081 DEBUG [M:0;d029b80fd32e:43289 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41234899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:17:13,081 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,43289,1733436971101 2024-12-05T22:17:13,081 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,43289,1733436971101; all regions closed. 2024-12-05T22:17:13,081 DEBUG [M:0;d029b80fd32e:43289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:17:13,081 DEBUG [M:0;d029b80fd32e:43289 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T22:17:13,081 DEBUG [M:0;d029b80fd32e:43289 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T22:17:13,081 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T22:17:13,081 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436971319 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733436971319,5,FailOnTimeoutGroup] 2024-12-05T22:17:13,081 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436971320 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733436971320,5,FailOnTimeoutGroup] 2024-12-05T22:17:13,081 INFO [M:0;d029b80fd32e:43289 {}] hbase.ChoreService(370): Chore service for: master/d029b80fd32e:0 had [] on shutdown 2024-12-05T22:17:13,081 DEBUG [M:0;d029b80fd32e:43289 {}] master.HMaster(1733): Stopping service threads 2024-12-05T22:17:13,081 INFO [M:0;d029b80fd32e:43289 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T22:17:13,081 INFO [M:0;d029b80fd32e:43289 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T22:17:13,081 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T22:17:13,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T22:17:13,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:13,082 DEBUG [M:0;d029b80fd32e:43289 {}] zookeeper.ZKUtil(347): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T22:17:13,082 WARN [M:0;d029b80fd32e:43289 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T22:17:13,082 INFO [M:0;d029b80fd32e:43289 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-05T22:17:13,082 INFO [M:0;d029b80fd32e:43289 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T22:17:13,082 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:17:13,082 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:17:13,082 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:17:13,082 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:17:13,082 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:17:13,083 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:17:13,083 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=65.00 KB heapSize=81.57 KB 2024-12-05T22:17:13,099 DEBUG [M:0;d029b80fd32e:43289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b835b417ab444040b9702f2929c79564 is 82, key is hbase:meta,,1/info:regioninfo/1733436972095/Put/seqid=0 2024-12-05T22:17:13,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741851_1027 (size=5672) 2024-12-05T22:17:13,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741851_1027 (size=5672) 2024-12-05T22:17:13,104 INFO [M:0;d029b80fd32e:43289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b835b417ab444040b9702f2929c79564 2024-12-05T22:17:13,123 DEBUG [M:0;d029b80fd32e:43289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5f49ddfe7a2b49f79a02eba33cbaad4a is 797, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733436973108/Put/seqid=0 2024-12-05T22:17:13,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741852_1028 (size=8348) 2024-12-05T22:17:13,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741852_1028 (size=8348) 2024-12-05T22:17:13,129 INFO [M:0;d029b80fd32e:43289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.40 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5f49ddfe7a2b49f79a02eba33cbaad4a 2024-12-05T22:17:13,134 INFO [M:0;d029b80fd32e:43289 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5f49ddfe7a2b49f79a02eba33cbaad4a 2024-12-05T22:17:13,151 DEBUG [M:0;d029b80fd32e:43289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3a83f809915c49309b31c66ac59c8e7b is 69, key is d029b80fd32e,33147,1733436971160/rs:state/1733436971430/Put/seqid=0 2024-12-05T22:17:13,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741853_1029 (size=5156) 2024-12-05T22:17:13,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741853_1029 (size=5156) 2024-12-05T22:17:13,161 INFO [M:0;d029b80fd32e:43289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3a83f809915c49309b31c66ac59c8e7b 2024-12-05T22:17:13,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:17:13,179 INFO [RS:0;d029b80fd32e:33147 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,33147,1733436971160; zookeeper connection closed. 2024-12-05T22:17:13,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33147-0x10184b302dc0001, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:17:13,179 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@64dda8be {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@64dda8be 2024-12-05T22:17:13,180 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T22:17:13,181 DEBUG [M:0;d029b80fd32e:43289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e290ebc6cb064dd484d970b33e720761 is 52, key is load_balancer_on/state:d/1733436972693/Put/seqid=0 2024-12-05T22:17:13,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741854_1030 (size=5056) 2024-12-05T22:17:13,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741854_1030 (size=5056) 2024-12-05T22:17:13,186 INFO [M:0;d029b80fd32e:43289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e290ebc6cb064dd484d970b33e720761 2024-12-05T22:17:13,192 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b835b417ab444040b9702f2929c79564 as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b835b417ab444040b9702f2929c79564 2024-12-05T22:17:13,200 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b835b417ab444040b9702f2929c79564, entries=8, sequenceid=184, filesize=5.5 K 2024-12-05T22:17:13,201 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5f49ddfe7a2b49f79a02eba33cbaad4a as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5f49ddfe7a2b49f79a02eba33cbaad4a 2024-12-05T22:17:13,208 INFO [M:0;d029b80fd32e:43289 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5f49ddfe7a2b49f79a02eba33cbaad4a 2024-12-05T22:17:13,208 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5f49ddfe7a2b49f79a02eba33cbaad4a, entries=21, sequenceid=184, filesize=8.2 K 2024-12-05T22:17:13,209 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3a83f809915c49309b31c66ac59c8e7b as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3a83f809915c49309b31c66ac59c8e7b 2024-12-05T22:17:13,214 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3a83f809915c49309b31c66ac59c8e7b, entries=1, sequenceid=184, filesize=5.0 K 2024-12-05T22:17:13,215 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e290ebc6cb064dd484d970b33e720761 as hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e290ebc6cb064dd484d970b33e720761 2024-12-05T22:17:13,219 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:36979/user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e290ebc6cb064dd484d970b33e720761, entries=1, sequenceid=184, filesize=4.9 K 2024-12-05T22:17:13,220 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HRegion(3040): Finished flush of dataSize ~65.00 KB/66562, heapSize ~81.51 KB/83464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 137ms, sequenceid=184, compaction requested=false 2024-12-05T22:17:13,221 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:17:13,221 DEBUG [M:0;d029b80fd32e:43289 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:17:13,222 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/7490293a-9bb0-8e6c-7cd9-57130a714ec3/MasterData/WALs/d029b80fd32e,43289,1733436971101 2024-12-05T22:17:13,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40371 is added to blk_1073741830_1006 (size=79083) 2024-12-05T22:17:13,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39331 is added to blk_1073741830_1006 (size=79083) 2024-12-05T22:17:13,224 INFO [M:0;d029b80fd32e:43289 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-05T22:17:13,224 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:17:13,224 INFO [M:0;d029b80fd32e:43289 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:43289 2024-12-05T22:17:13,227 DEBUG [M:0;d029b80fd32e:43289 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/d029b80fd32e,43289,1733436971101 already deleted, retry=false 2024-12-05T22:17:13,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:17:13,329 INFO [M:0;d029b80fd32e:43289 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,43289,1733436971101; zookeeper connection closed. 2024-12-05T22:17:13,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43289-0x10184b302dc0000, quorum=127.0.0.1:50103, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:17:13,331 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b6aeddc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:17:13,331 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@607e1d6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:17:13,332 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:17:13,332 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@214f83d4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:17:13,332 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29d5bebf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.log.dir/,STOPPED} 2024-12-05T22:17:13,333 WARN [BP-909378404-172.17.0.2-1733436970301 heartbeating to localhost/127.0.0.1:36979 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:17:13,333 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:17:13,333 WARN [BP-909378404-172.17.0.2-1733436970301 heartbeating to localhost/127.0.0.1:36979 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-909378404-172.17.0.2-1733436970301 (Datanode Uuid b4d5cc32-ac0f-47d9-a915-357b0155f981) service to localhost/127.0.0.1:36979 2024-12-05T22:17:13,333 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:17:13,334 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/dfs/data/data3/current/BP-909378404-172.17.0.2-1733436970301 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:17:13,334 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/dfs/data/data4/current/BP-909378404-172.17.0.2-1733436970301 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:17:13,334 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:17:13,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a0d04be{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:17:13,337 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@634fdd28{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:17:13,337 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:17:13,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18c4ea87{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:17:13,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d858d58{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.log.dir/,STOPPED} 2024-12-05T22:17:13,339 WARN [BP-909378404-172.17.0.2-1733436970301 heartbeating to localhost/127.0.0.1:36979 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:17:13,339 WARN [BP-909378404-172.17.0.2-1733436970301 heartbeating to localhost/127.0.0.1:36979 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-909378404-172.17.0.2-1733436970301 (Datanode Uuid 9b840b89-7497-4486-8b0e-8b9fa804c0b2) service to localhost/127.0.0.1:36979 2024-12-05T22:17:13,339 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:17:13,339 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:17:13,340 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/dfs/data/data1/current/BP-909378404-172.17.0.2-1733436970301 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:17:13,340 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/cluster_a80f056d-f36a-ef38-70a2-62ac0bc3c998/dfs/data/data2/current/BP-909378404-172.17.0.2-1733436970301 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:17:13,340 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:17:13,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@903456d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:17:13,348 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@58429b8e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:17:13,348 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:17:13,348 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f70eed0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:17:13,348 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20b89acb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.log.dir/,STOPPED} 2024-12-05T22:17:13,357 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-05T22:17:13,382 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-05T22:17:13,394 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=110 (was 100) - Thread LEAK? -, OpenFileDescriptor=466 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=348 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=2810 (was 2914) 2024-12-05T22:17:13,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=111, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=348, ProcessCount=11, AvailableMemoryMB=2810 2024-12-05T22:17:13,412 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.log.dir so I do NOT create it in target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b05dccf8-78e7-b2e3-63ff-5e469c5f4957/hadoop.tmp.dir so I do NOT create it in target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91, deleteOnExit=true 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/test.cache.data in system properties and HBase conf 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.log.dir in system properties and HBase conf 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T22:17:13,413 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-05T22:17:13,414 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T22:17:13,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:17:13,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:17:13,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T22:17:13,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:17:13,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T22:17:13,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T22:17:13,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:17:13,414 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:17:13,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T22:17:13,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/nfs.dump.dir in system properties and HBase conf 2024-12-05T22:17:13,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/java.io.tmpdir in system properties and HBase conf 2024-12-05T22:17:13,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:17:13,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T22:17:13,415 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T22:17:13,433 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:17:13,443 INFO [regionserver/d029b80fd32e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:17:13,497 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:13,515 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:17:13,520 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:17:13,522 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:17:13,522 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:17:13,522 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:17:13,522 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:17:13,523 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52e6c5a0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:17:13,523 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3fd2286c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:17:13,659 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1cd1d5c9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/java.io.tmpdir/jetty-localhost-35561-hadoop-hdfs-3_4_1-tests_jar-_-any-9499284203097750568/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:17:13,660 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ff97e6a{HTTP/1.1, (http/1.1)}{localhost:35561} 2024-12-05T22:17:13,660 INFO [Time-limited test {}] server.Server(415): Started @287378ms 2024-12-05T22:17:13,679 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:17:13,781 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:17:13,785 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:17:13,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:17:13,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:17:13,786 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:17:13,789 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@65cd88a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:17:13,789 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29238c79{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:17:13,931 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4be9feb9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/java.io.tmpdir/jetty-localhost-42923-hadoop-hdfs-3_4_1-tests_jar-_-any-14385282783814168662/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:17:13,932 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@73d21e1d{HTTP/1.1, (http/1.1)}{localhost:42923} 2024-12-05T22:17:13,932 INFO [Time-limited test {}] server.Server(415): Started @287649ms 2024-12-05T22:17:13,933 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:17:13,999 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:17:14,003 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:17:14,009 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:17:14,009 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:17:14,009 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:17:14,013 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@548d4f02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:17:14,013 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4105b643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:17:14,119 WARN [Thread-1693 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/dfs/data/data1/current/BP-495755302-172.17.0.2-1733437033442/current, will proceed with Du for space computation calculation, 2024-12-05T22:17:14,128 WARN [Thread-1694 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/dfs/data/data2/current/BP-495755302-172.17.0.2-1733437033442/current, will proceed with Du for space computation calculation, 2024-12-05T22:17:14,186 WARN [Thread-1672 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:17:14,193 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1e6d7f99{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/java.io.tmpdir/jetty-localhost-41409-hadoop-hdfs-3_4_1-tests_jar-_-any-16084677923612669384/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:17:14,195 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x775031bf25624405 with lease ID 0xf4be07a5ce16e989: Processing first storage report for DS-134f7dfe-dc7f-4c23-a929-9f07251caadd from datanode DatanodeRegistration(127.0.0.1:46287, datanodeUuid=ef54fe69-bf95-4af9-81cb-e98911855bf1, infoPort=44117, infoSecurePort=0, ipcPort=37057, storageInfo=lv=-57;cid=testClusterID;nsid=285162862;c=1733437033442) 2024-12-05T22:17:14,195 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x775031bf25624405 with lease ID 0xf4be07a5ce16e989: from storage DS-134f7dfe-dc7f-4c23-a929-9f07251caadd node DatanodeRegistration(127.0.0.1:46287, datanodeUuid=ef54fe69-bf95-4af9-81cb-e98911855bf1, infoPort=44117, infoSecurePort=0, ipcPort=37057, storageInfo=lv=-57;cid=testClusterID;nsid=285162862;c=1733437033442), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:17:14,195 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x775031bf25624405 with lease ID 0xf4be07a5ce16e989: Processing first storage report for DS-7574f038-8f64-47f8-9471-bcd9f358f253 from datanode DatanodeRegistration(127.0.0.1:46287, datanodeUuid=ef54fe69-bf95-4af9-81cb-e98911855bf1, infoPort=44117, infoSecurePort=0, ipcPort=37057, storageInfo=lv=-57;cid=testClusterID;nsid=285162862;c=1733437033442) 2024-12-05T22:17:14,195 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x775031bf25624405 with lease ID 0xf4be07a5ce16e989: from storage DS-7574f038-8f64-47f8-9471-bcd9f358f253 node DatanodeRegistration(127.0.0.1:46287, datanodeUuid=ef54fe69-bf95-4af9-81cb-e98911855bf1, infoPort=44117, infoSecurePort=0, ipcPort=37057, storageInfo=lv=-57;cid=testClusterID;nsid=285162862;c=1733437033442), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:17:14,196 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@710c1196{HTTP/1.1, (http/1.1)}{localhost:41409} 2024-12-05T22:17:14,196 INFO [Time-limited test {}] server.Server(415): Started @287914ms 2024-12-05T22:17:14,198 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:17:14,349 WARN [Thread-1719 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/dfs/data/data3/current/BP-495755302-172.17.0.2-1733437033442/current, will proceed with Du for space computation calculation, 2024-12-05T22:17:14,349 WARN [Thread-1720 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/dfs/data/data4/current/BP-495755302-172.17.0.2-1733437033442/current, will proceed with Du for space computation calculation, 2024-12-05T22:17:14,385 WARN [Thread-1708 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:17:14,390 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x48348db57e5c6ec2 with lease ID 0xf4be07a5ce16e98a: Processing first storage report for DS-6a9f7e90-0078-4241-8f7a-b0e328135cf5 from datanode DatanodeRegistration(127.0.0.1:36311, datanodeUuid=4e8ceff1-9976-4027-bd50-ad575a0acff6, infoPort=45521, infoSecurePort=0, ipcPort=43579, storageInfo=lv=-57;cid=testClusterID;nsid=285162862;c=1733437033442) 2024-12-05T22:17:14,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x48348db57e5c6ec2 with lease ID 0xf4be07a5ce16e98a: from storage DS-6a9f7e90-0078-4241-8f7a-b0e328135cf5 node DatanodeRegistration(127.0.0.1:36311, datanodeUuid=4e8ceff1-9976-4027-bd50-ad575a0acff6, infoPort=45521, infoSecurePort=0, ipcPort=43579, storageInfo=lv=-57;cid=testClusterID;nsid=285162862;c=1733437033442), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:17:14,391 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x48348db57e5c6ec2 with lease ID 0xf4be07a5ce16e98a: Processing first storage report for DS-f4905bb7-881d-4e2d-892e-31f113d38918 from datanode DatanodeRegistration(127.0.0.1:36311, datanodeUuid=4e8ceff1-9976-4027-bd50-ad575a0acff6, infoPort=45521, infoSecurePort=0, ipcPort=43579, storageInfo=lv=-57;cid=testClusterID;nsid=285162862;c=1733437033442) 2024-12-05T22:17:14,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x48348db57e5c6ec2 with lease ID 0xf4be07a5ce16e98a: from storage DS-f4905bb7-881d-4e2d-892e-31f113d38918 node DatanodeRegistration(127.0.0.1:36311, datanodeUuid=4e8ceff1-9976-4027-bd50-ad575a0acff6, infoPort=45521, infoSecurePort=0, ipcPort=43579, storageInfo=lv=-57;cid=testClusterID;nsid=285162862;c=1733437033442), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:17:14,487 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1 2024-12-05T22:17:14,493 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/zookeeper_0, clientPort=60973, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T22:17:14,494 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=60973 2024-12-05T22:17:14,495 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:17:14,496 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:17:14,497 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:14,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:17:14,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:17:14,539 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc with version=8 2024-12-05T22:17:14,539 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/hbase-staging 2024-12-05T22:17:14,542 INFO [Time-limited test {}] client.ConnectionUtils(129): master/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:17:14,542 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:17:14,542 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:17:14,542 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:17:14,542 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:17:14,542 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:17:14,543 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:17:14,543 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:17:14,544 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:41859 2024-12-05T22:17:14,545 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:17:14,547 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:17:14,549 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:41859 connecting to ZooKeeper ensemble=127.0.0.1:60973 2024-12-05T22:17:14,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418590x0, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:17:14,565 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41859-0x10184b3fa890000 connected 2024-12-05T22:17:14,596 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:17:14,597 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:17:14,598 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:17:14,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41859 2024-12-05T22:17:14,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41859 2024-12-05T22:17:14,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41859 2024-12-05T22:17:14,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41859 2024-12-05T22:17:14,612 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41859 2024-12-05T22:17:14,613 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc, hbase.cluster.distributed=false 2024-12-05T22:17:14,636 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:17:14,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:17:14,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:17:14,636 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:17:14,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:17:14,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:17:14,636 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:17:14,636 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:17:14,638 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38777 2024-12-05T22:17:14,638 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T22:17:14,641 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T22:17:14,642 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:17:14,645 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:17:14,649 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:38777 connecting to ZooKeeper ensemble=127.0.0.1:60973 2024-12-05T22:17:14,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387770x0, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:17:14,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387770x0, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:17:14,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387770x0, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:17:14,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387770x0, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:17:14,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38777-0x10184b3fa890001 connected 2024-12-05T22:17:14,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38777 2024-12-05T22:17:14,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38777 2024-12-05T22:17:14,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38777 2024-12-05T22:17:14,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38777 2024-12-05T22:17:14,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38777 2024-12-05T22:17:14,688 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/d029b80fd32e,41859,1733437034541 2024-12-05T22:17:14,695 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d029b80fd32e,41859,1733437034541 2024-12-05T22:17:14,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:17:14,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:17:14,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:17:14,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,697 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:17:14,698 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d029b80fd32e,41859,1733437034541 from backup master directory 2024-12-05T22:17:14,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:17:14,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d029b80fd32e,41859,1733437034541 2024-12-05T22:17:14,702 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:17:14,704 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:17:14,705 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d029b80fd32e,41859,1733437034541 2024-12-05T22:17:14,709 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d029b80fd32e:41859 2024-12-05T22:17:14,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:17:14,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:17:14,731 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/hbase.id with ID: da685bd0-9f9f-42c1-ae4d-a480773ec32a 2024-12-05T22:17:14,751 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:17:14,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:17:14,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:17:14,781 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:17:14,782 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T22:17:14,783 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:17:14,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:17:14,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:17:14,791 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store 2024-12-05T22:17:14,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:17:14,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:17:14,798 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:14,798 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:17:14,798 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:17:14,798 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:17:14,798 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:17:14,798 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:17:14,798 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:17:14,798 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:17:14,799 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/.initializing 2024-12-05T22:17:14,799 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/WALs/d029b80fd32e,41859,1733437034541 2024-12-05T22:17:14,801 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C41859%2C1733437034541, suffix=, logDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/WALs/d029b80fd32e,41859,1733437034541, archiveDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/oldWALs, maxLogs=10 2024-12-05T22:17:14,802 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C41859%2C1733437034541.1733437034802 2024-12-05T22:17:14,806 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/WALs/d029b80fd32e,41859,1733437034541/d029b80fd32e%2C41859%2C1733437034541.1733437034802 2024-12-05T22:17:14,806 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45521:45521),(127.0.0.1/127.0.0.1:44117:44117)] 2024-12-05T22:17:14,806 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:17:14,806 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:14,807 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:17:14,807 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:17:14,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:17:14,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T22:17:14,809 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:14,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:17:14,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:17:14,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T22:17:14,811 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:14,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:17:14,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:17:14,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T22:17:14,813 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:14,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:17:14,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:17:14,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T22:17:14,815 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:14,815 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:17:14,816 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:17:14,817 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:17:14,818 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T22:17:14,819 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:17:14,821 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:17:14,821 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=877243, jitterRate=0.11547280848026276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T22:17:14,822 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:17:14,822 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T22:17:14,826 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac1b441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:17:14,827 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-05T22:17:14,827 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T22:17:14,827 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T22:17:14,827 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T22:17:14,828 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T22:17:14,828 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-05T22:17:14,828 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T22:17:14,830 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T22:17:14,831 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T22:17:14,832 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-05T22:17:14,832 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T22:17:14,832 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T22:17:14,833 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-05T22:17:14,833 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T22:17:14,834 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T22:17:14,836 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-05T22:17:14,837 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T22:17:14,838 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T22:17:14,839 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T22:17:14,840 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T22:17:14,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:17:14,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:17:14,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,842 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=d029b80fd32e,41859,1733437034541, sessionid=0x10184b3fa890000, setting cluster-up flag (Was=false) 2024-12-05T22:17:14,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,851 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T22:17:14,851 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,41859,1733437034541 2024-12-05T22:17:14,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:14,860 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T22:17:14,861 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,41859,1733437034541 2024-12-05T22:17:14,867 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-05T22:17:14,867 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-05T22:17:14,867 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T22:17:14,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d029b80fd32e,41859,1733437034541 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T22:17:14,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:17:14,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:17:14,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:17:14,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:17:14,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d029b80fd32e:0, corePoolSize=10, maxPoolSize=10 2024-12-05T22:17:14,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:17:14,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,869 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733437064869 2024-12-05T22:17:14,869 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T22:17:14,870 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T22:17:14,870 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T22:17:14,870 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T22:17:14,870 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T22:17:14,870 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T22:17:14,870 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:17:14,870 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-05T22:17:14,872 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,872 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:14,872 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:17:14,873 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T22:17:14,873 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T22:17:14,873 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T22:17:14,874 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T22:17:14,874 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T22:17:14,874 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733437034874,5,FailOnTimeoutGroup] 2024-12-05T22:17:14,874 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733437034874,5,FailOnTimeoutGroup] 2024-12-05T22:17:14,874 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,874 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T22:17:14,874 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,875 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:17:14,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:17:14,882 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-05T22:17:14,882 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc 2024-12-05T22:17:14,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:17:14,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:17:14,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:14,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:17:14,896 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:17:14,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:14,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:17:14,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:17:14,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:17:14,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:14,898 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:17:14,898 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:17:14,900 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:17:14,900 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:14,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:17:14,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740 2024-12-05T22:17:14,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740 2024-12-05T22:17:14,901 DEBUG [RS:0;d029b80fd32e:38777 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d029b80fd32e:38777 2024-12-05T22:17:14,902 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1008): ClusterId : da685bd0-9f9f-42c1-ae4d-a480773ec32a 2024-12-05T22:17:14,902 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T22:17:14,903 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:17:14,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:17:14,904 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T22:17:14,904 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T22:17:14,906 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T22:17:14,907 DEBUG [RS:0;d029b80fd32e:38777 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c50e67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:17:14,907 DEBUG [RS:0;d029b80fd32e:38777 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@98252a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:17:14,907 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-05T22:17:14,907 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-05T22:17:14,907 DEBUG [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-05T22:17:14,908 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(3073): reportForDuty to master=d029b80fd32e,41859,1733437034541 with isa=d029b80fd32e/172.17.0.2:38777, startcode=1733437034635 2024-12-05T22:17:14,908 DEBUG [RS:0;d029b80fd32e:38777 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:17:14,910 INFO [RS-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57471, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:17:14,910 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41859 {}] master.ServerManager(332): Checking decommissioned status of RegionServer d029b80fd32e,38777,1733437034635 2024-12-05T22:17:14,911 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41859 {}] master.ServerManager(486): Registering regionserver=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:14,912 DEBUG [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc 2024-12-05T22:17:14,912 DEBUG [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:46345 2024-12-05T22:17:14,912 DEBUG [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-05T22:17:14,914 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:17:14,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:17:14,915 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=745421, jitterRate=-0.05214942991733551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:17:14,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:17:14,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:17:14,915 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:17:14,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:17:14,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:17:14,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:17:14,916 DEBUG [RS:0;d029b80fd32e:38777 {}] zookeeper.ZKUtil(111): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d029b80fd32e,38777,1733437034635 2024-12-05T22:17:14,916 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:17:14,916 WARN [RS:0;d029b80fd32e:38777 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:17:14,916 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d029b80fd32e,38777,1733437034635] 2024-12-05T22:17:14,916 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:17:14,916 INFO [RS:0;d029b80fd32e:38777 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:17:14,916 DEBUG [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635 2024-12-05T22:17:14,917 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:17:14,917 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-05T22:17:14,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T22:17:14,918 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T22:17:14,920 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T22:17:14,922 DEBUG [RS:0;d029b80fd32e:38777 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-05T22:17:14,922 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T22:17:14,923 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T22:17:14,924 INFO [RS:0;d029b80fd32e:38777 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T22:17:14,924 INFO [RS:0;d029b80fd32e:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,924 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-05T22:17:14,925 INFO [RS:0;d029b80fd32e:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,925 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,925 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,925 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,925 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,925 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,925 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:17:14,925 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,926 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,926 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,926 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,926 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:17:14,926 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:17:14,926 DEBUG [RS:0;d029b80fd32e:38777 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:17:14,926 INFO [RS:0;d029b80fd32e:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,926 INFO [RS:0;d029b80fd32e:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,926 INFO [RS:0;d029b80fd32e:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,926 INFO [RS:0;d029b80fd32e:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,926 INFO [RS:0;d029b80fd32e:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,38777,1733437034635-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:17:14,941 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T22:17:14,942 INFO [RS:0;d029b80fd32e:38777 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,38777,1733437034635-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:14,956 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.Replication(204): d029b80fd32e,38777,1733437034635 started 2024-12-05T22:17:14,956 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1767): Serving as d029b80fd32e,38777,1733437034635, RpcServer on d029b80fd32e/172.17.0.2:38777, sessionid=0x10184b3fa890001 2024-12-05T22:17:14,956 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T22:17:14,957 DEBUG [RS:0;d029b80fd32e:38777 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d029b80fd32e,38777,1733437034635 2024-12-05T22:17:14,957 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,38777,1733437034635' 2024-12-05T22:17:14,957 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T22:17:14,957 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T22:17:14,957 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T22:17:14,957 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T22:17:14,957 DEBUG [RS:0;d029b80fd32e:38777 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d029b80fd32e,38777,1733437034635 2024-12-05T22:17:14,957 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,38777,1733437034635' 2024-12-05T22:17:14,957 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T22:17:14,958 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T22:17:14,958 DEBUG [RS:0;d029b80fd32e:38777 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T22:17:14,958 INFO [RS:0;d029b80fd32e:38777 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T22:17:14,958 INFO [RS:0;d029b80fd32e:38777 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T22:17:15,061 INFO [RS:0;d029b80fd32e:38777 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C38777%2C1733437034635, suffix=, logDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635, archiveDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/oldWALs, maxLogs=32 2024-12-05T22:17:15,062 INFO [RS:0;d029b80fd32e:38777 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38777%2C1733437034635.1733437035062 2024-12-05T22:17:15,071 WARN [d029b80fd32e:41859 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-05T22:17:15,116 INFO [RS:0;d029b80fd32e:38777 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.1733437035062 2024-12-05T22:17:15,117 DEBUG [RS:0;d029b80fd32e:38777 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45521:45521),(127.0.0.1/127.0.0.1:44117:44117)] 2024-12-05T22:17:15,321 DEBUG [d029b80fd32e:41859 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T22:17:15,321 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:15,322 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,38777,1733437034635, state=OPENING 2024-12-05T22:17:15,324 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T22:17:15,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:15,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:15,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:17:15,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:17:15,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=d029b80fd32e,38777,1733437034635}] 2024-12-05T22:17:15,479 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d029b80fd32e,38777,1733437034635 2024-12-05T22:17:15,479 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T22:17:15,481 INFO [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35294, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T22:17:15,488 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-05T22:17:15,489 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:17:15,491 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C38777%2C1733437034635.meta, suffix=.meta, logDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635, archiveDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/oldWALs, maxLogs=32 2024-12-05T22:17:15,492 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38777%2C1733437034635.meta.1733437035492.meta 2024-12-05T22:17:15,498 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:15,501 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.meta.1733437035492.meta 2024-12-05T22:17:15,501 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44117:44117),(127.0.0.1/127.0.0.1:45521:45521)] 2024-12-05T22:17:15,501 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:17:15,501 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T22:17:15,501 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T22:17:15,502 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T22:17:15,502 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T22:17:15,502 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:15,502 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-05T22:17:15,502 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-05T22:17:15,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:17:15,504 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:17:15,504 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:15,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:17:15,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:17:15,505 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:17:15,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:15,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:17:15,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:17:15,506 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:17:15,506 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:15,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:17:15,507 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740 2024-12-05T22:17:15,508 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740 2024-12-05T22:17:15,509 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:17:15,510 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:17:15,511 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=847998, jitterRate=0.07828640937805176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:17:15,511 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:17:15,512 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733437035478 2024-12-05T22:17:15,514 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T22:17:15,514 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-05T22:17:15,514 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:15,515 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,38777,1733437034635, state=OPEN 2024-12-05T22:17:15,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:17:15,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:17:15,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:17:15,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:17:15,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T22:17:15,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=d029b80fd32e,38777,1733437034635 in 211 msec 2024-12-05T22:17:15,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T22:17:15,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 622 msec 2024-12-05T22:17:15,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 678 msec 2024-12-05T22:17:15,542 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733437035542, completionTime=-1 2024-12-05T22:17:15,542 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T22:17:15,542 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-05T22:17:15,543 DEBUG [hconnection-0x5f1fecba-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:17:15,544 INFO [RS-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35296, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:17:15,545 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-05T22:17:15,545 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733437095545 2024-12-05T22:17:15,545 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733437155545 2024-12-05T22:17:15,545 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec 2024-12-05T22:17:15,550 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,41859,1733437034541-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:15,550 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,41859,1733437034541-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:15,550 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,41859,1733437034541-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:15,550 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d029b80fd32e:41859, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:15,550 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:15,551 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-05T22:17:15,551 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:17:15,552 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-05T22:17:15,552 DEBUG [master/d029b80fd32e:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-05T22:17:15,553 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:17:15,553 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:15,554 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:17:15,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:17:15,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:17:15,565 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 883cc4622e8d8175d333f4fa3e2982b4, NAME => 'hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc 2024-12-05T22:17:15,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:17:15,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:17:15,572 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:15,572 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 883cc4622e8d8175d333f4fa3e2982b4, disabling compactions & flushes 2024-12-05T22:17:15,572 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:17:15,572 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:17:15,572 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. after waiting 0 ms 2024-12-05T22:17:15,572 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:17:15,572 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:17:15,572 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 883cc4622e8d8175d333f4fa3e2982b4: 2024-12-05T22:17:15,573 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:17:15,574 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733437035573"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733437035573"}]},"ts":"1733437035573"} 2024-12-05T22:17:15,575 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:17:15,576 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:17:15,576 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733437035576"}]},"ts":"1733437035576"} 2024-12-05T22:17:15,577 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-05T22:17:15,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=883cc4622e8d8175d333f4fa3e2982b4, ASSIGN}] 2024-12-05T22:17:15,586 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=883cc4622e8d8175d333f4fa3e2982b4, ASSIGN 2024-12-05T22:17:15,587 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=883cc4622e8d8175d333f4fa3e2982b4, ASSIGN; state=OFFLINE, location=d029b80fd32e,38777,1733437034635; forceNewPlan=false, retain=false 2024-12-05T22:17:15,737 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=883cc4622e8d8175d333f4fa3e2982b4, regionState=OPENING, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:15,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 883cc4622e8d8175d333f4fa3e2982b4, server=d029b80fd32e,38777,1733437034635}] 2024-12-05T22:17:15,892 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d029b80fd32e,38777,1733437034635 2024-12-05T22:17:15,896 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:17:15,896 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 883cc4622e8d8175d333f4fa3e2982b4, NAME => 'hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:17:15,896 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 883cc4622e8d8175d333f4fa3e2982b4 2024-12-05T22:17:15,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:15,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 883cc4622e8d8175d333f4fa3e2982b4 2024-12-05T22:17:15,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 883cc4622e8d8175d333f4fa3e2982b4 2024-12-05T22:17:15,898 INFO [StoreOpener-883cc4622e8d8175d333f4fa3e2982b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 883cc4622e8d8175d333f4fa3e2982b4 2024-12-05T22:17:15,899 INFO [StoreOpener-883cc4622e8d8175d333f4fa3e2982b4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883cc4622e8d8175d333f4fa3e2982b4 columnFamilyName info 2024-12-05T22:17:15,899 DEBUG [StoreOpener-883cc4622e8d8175d333f4fa3e2982b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:15,900 INFO [StoreOpener-883cc4622e8d8175d333f4fa3e2982b4-1 {}] regionserver.HStore(327): Store=883cc4622e8d8175d333f4fa3e2982b4/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:17:15,901 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/namespace/883cc4622e8d8175d333f4fa3e2982b4 2024-12-05T22:17:15,901 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/namespace/883cc4622e8d8175d333f4fa3e2982b4 2024-12-05T22:17:15,904 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 883cc4622e8d8175d333f4fa3e2982b4 2024-12-05T22:17:15,906 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/namespace/883cc4622e8d8175d333f4fa3e2982b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:17:15,907 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 883cc4622e8d8175d333f4fa3e2982b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=833751, jitterRate=0.06016974151134491}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:17:15,907 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 883cc4622e8d8175d333f4fa3e2982b4: 2024-12-05T22:17:15,908 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4., pid=6, masterSystemTime=1733437035892 2024-12-05T22:17:15,914 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:17:15,914 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:17:15,915 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=883cc4622e8d8175d333f4fa3e2982b4, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:15,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T22:17:15,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 883cc4622e8d8175d333f4fa3e2982b4, server=d029b80fd32e,38777,1733437034635 in 182 msec 2024-12-05T22:17:15,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T22:17:15,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=883cc4622e8d8175d333f4fa3e2982b4, ASSIGN in 340 msec 2024-12-05T22:17:15,930 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:17:15,931 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733437035930"}]},"ts":"1733437035930"} 2024-12-05T22:17:15,933 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-05T22:17:15,938 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:17:15,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 387 msec 2024-12-05T22:17:15,953 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-05T22:17:15,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:15,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:17:15,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:17:15,963 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-05T22:17:15,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:17:15,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 29 msec 2024-12-05T22:17:15,995 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-05T22:17:16,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:17:16,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 12 msec 2024-12-05T22:17:16,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-05T22:17:16,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-05T22:17:16,028 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.323sec 2024-12-05T22:17:16,028 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T22:17:16,028 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T22:17:16,028 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T22:17:16,028 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T22:17:16,028 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T22:17:16,028 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,41859,1733437034541-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:17:16,028 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,41859,1733437034541-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T22:17:16,032 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-05T22:17:16,032 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T22:17:16,032 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,41859,1733437034541-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:17:16,095 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x429e73de to 127.0.0.1:60973 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@64575e42 2024-12-05T22:17:16,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6870ea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:17:16,107 DEBUG [hconnection-0x6c85aea3-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:17:16,109 INFO [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35300, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:17:16,111 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=d029b80fd32e,41859,1733437034541 2024-12-05T22:17:16,112 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:17:16,117 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-05T22:17:16,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T22:17:16,121 INFO [RS-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50698, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T22:17:16,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41859 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-05T22:17:16,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41859 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-05T22:17:16,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41859 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:17:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41859 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-12-05T22:17:16,126 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:17:16,127 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:16,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41859 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 9 2024-12-05T22:17:16,129 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:17:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41859 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:17:16,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741837_1013 (size=381) 2024-12-05T22:17:16,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741837_1013 (size=381) 2024-12-05T22:17:16,498 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:16,543 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6166a9886eff2aa6d15d170434051ce7, NAME => 'TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc 2024-12-05T22:17:16,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741838_1014 (size=64) 2024-12-05T22:17:16,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741838_1014 (size=64) 2024-12-05T22:17:16,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:16,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing 6166a9886eff2aa6d15d170434051ce7, disabling compactions & flushes 2024-12-05T22:17:16,553 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:16,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:16,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. after waiting 0 ms 2024-12-05T22:17:16,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:16,553 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:16,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:16,554 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:17:16,554 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733437036554"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733437036554"}]},"ts":"1733437036554"} 2024-12-05T22:17:16,556 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:17:16,559 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:17:16,559 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733437036559"}]},"ts":"1733437036559"} 2024-12-05T22:17:16,560 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-12-05T22:17:16,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6166a9886eff2aa6d15d170434051ce7, ASSIGN}] 2024-12-05T22:17:16,567 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6166a9886eff2aa6d15d170434051ce7, ASSIGN 2024-12-05T22:17:16,568 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6166a9886eff2aa6d15d170434051ce7, ASSIGN; state=OFFLINE, location=d029b80fd32e,38777,1733437034635; forceNewPlan=false, retain=false 2024-12-05T22:17:16,719 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=6166a9886eff2aa6d15d170434051ce7, regionState=OPENING, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:16,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635}] 2024-12-05T22:17:16,874 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d029b80fd32e,38777,1733437034635 2024-12-05T22:17:16,879 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:16,879 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 6166a9886eff2aa6d15d170434051ce7, NAME => 'TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:17:16,880 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:16,880 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:16,880 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:16,880 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:16,884 INFO [StoreOpener-6166a9886eff2aa6d15d170434051ce7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:16,889 INFO [StoreOpener-6166a9886eff2aa6d15d170434051ce7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6166a9886eff2aa6d15d170434051ce7 columnFamilyName info 2024-12-05T22:17:16,889 DEBUG [StoreOpener-6166a9886eff2aa6d15d170434051ce7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:16,889 INFO [StoreOpener-6166a9886eff2aa6d15d170434051ce7-1 {}] regionserver.HStore(327): Store=6166a9886eff2aa6d15d170434051ce7/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:17:16,890 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:16,891 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:16,893 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:16,896 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:17:16,896 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 6166a9886eff2aa6d15d170434051ce7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=693925, jitterRate=-0.1176297515630722}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:17:16,897 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:16,898 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7., pid=11, masterSystemTime=1733437036874 2024-12-05T22:17:16,902 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:16,902 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:16,903 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=6166a9886eff2aa6d15d170434051ce7, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:16,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-12-05T22:17:16,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635 in 183 msec 2024-12-05T22:17:16,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T22:17:16,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6166a9886eff2aa6d15d170434051ce7, ASSIGN in 342 msec 2024-12-05T22:17:16,914 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:17:16,915 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733437036914"}]},"ts":"1733437036914"} 2024-12-05T22:17:16,917 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-12-05T22:17:16,920 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:17:16,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRolling in 797 msec 2024-12-05T22:17:17,499 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:17,935 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,935 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,935 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,935 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,936 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,936 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,960 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,961 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,961 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,961 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,961 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,962 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,966 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,966 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,967 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:17,969 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,475 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:17:18,476 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,476 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,476 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,476 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,477 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,477 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,499 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:18,504 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,505 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,505 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,506 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,506 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,506 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,512 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,512 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,512 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:18,516 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:19,500 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:20,501 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:20,922 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T22:17:20,923 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-05T22:17:20,923 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-12-05T22:17:21,501 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:22,502 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:22,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-12-05T22:17:22,541 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-05T22:17:22,542 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-05T22:17:23,503 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:24,504 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:25,504 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41859 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T22:17:26,131 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling, procId: 9 completed 2024-12-05T22:17:26,134 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRolling 2024-12-05T22:17:26,134 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:26,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 6166a9886eff2aa6d15d170434051ce7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:17:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T22:17:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35300 deadline: 1733437056183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:26,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/23f95331412a4fca92083b2b926118b2 is 1080, key is row0001/info:/1733437046137/Put/seqid=0 2024-12-05T22:17:26,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741839_1015 (size=12509) 2024-12-05T22:17:26,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741839_1015 (size=12509) 2024-12-05T22:17:26,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/23f95331412a4fca92083b2b926118b2 2024-12-05T22:17:26,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/23f95331412a4fca92083b2b926118b2 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/23f95331412a4fca92083b2b926118b2 2024-12-05T22:17:26,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/23f95331412a4fca92083b2b926118b2, entries=7, sequenceid=11, filesize=12.2 K 2024-12-05T22:17:26,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 6166a9886eff2aa6d15d170434051ce7 in 86ms, sequenceid=11, compaction requested=false 2024-12-05T22:17:26,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:26,505 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:27,505 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:28,044 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:17:28,045 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,045 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,046 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,046 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,046 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,046 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,064 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,064 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,065 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,065 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,065 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,065 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,068 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,069 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,069 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,072 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:28,506 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:29,507 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:30,507 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:31,508 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:32,509 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:32,937 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=1, created chunk count=15, reused chunk count=36, reuseRatio=70.59% 2024-12-05T22:17:32,937 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-12-05T22:17:33,509 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:34,510 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:35,511 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:36,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 6166a9886eff2aa6d15d170434051ce7 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-05T22:17:36,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/e94e4820241f4dfea2d7b51c8544a389 is 1080, key is row0008/info:/1733437046150/Put/seqid=0 2024-12-05T22:17:36,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741840_1016 (size=29761) 2024-12-05T22:17:36,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741840_1016 (size=29761) 2024-12-05T22:17:36,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/e94e4820241f4dfea2d7b51c8544a389 2024-12-05T22:17:36,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/e94e4820241f4dfea2d7b51c8544a389 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/e94e4820241f4dfea2d7b51c8544a389 2024-12-05T22:17:36,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/e94e4820241f4dfea2d7b51c8544a389, entries=23, sequenceid=37, filesize=29.1 K 2024-12-05T22:17:36,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 6166a9886eff2aa6d15d170434051ce7 in 24ms, sequenceid=37, compaction requested=false 2024-12-05T22:17:36,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:36,264 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=41.3 K, sizeToCheck=16.0 K 2024-12-05T22:17:36,264 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:17:36,264 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/e94e4820241f4dfea2d7b51c8544a389 because midkey is the same as first or last row 2024-12-05T22:17:36,511 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:37,512 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:38,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 6166a9886eff2aa6d15d170434051ce7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:17:38,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/03c40a367cce4929b9a61815d9680fb8 is 1080, key is row0031/info:/1733437056240/Put/seqid=0 2024-12-05T22:17:38,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741841_1017 (size=12509) 2024-12-05T22:17:38,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741841_1017 (size=12509) 2024-12-05T22:17:38,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/03c40a367cce4929b9a61815d9680fb8 2024-12-05T22:17:38,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/03c40a367cce4929b9a61815d9680fb8 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/03c40a367cce4929b9a61815d9680fb8 2024-12-05T22:17:38,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T22:17:38,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/03c40a367cce4929b9a61815d9680fb8, entries=7, sequenceid=47, filesize=12.2 K 2024-12-05T22:17:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35300 deadline: 1733437068271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:38,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 6166a9886eff2aa6d15d170434051ce7 in 24ms, sequenceid=47, compaction requested=true 2024-12-05T22:17:38,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:38,273 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=53.5 K, sizeToCheck=16.0 K 2024-12-05T22:17:38,273 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:17:38,273 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/e94e4820241f4dfea2d7b51c8544a389 because midkey is the same as first or last row 2024-12-05T22:17:38,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6166a9886eff2aa6d15d170434051ce7:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:17:38,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:17:38,273 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:17:38,275 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:17:38,275 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1540): 6166a9886eff2aa6d15d170434051ce7/info is initiating minor compaction (all files) 2024-12-05T22:17:38,275 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 6166a9886eff2aa6d15d170434051ce7/info in TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:38,275 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/23f95331412a4fca92083b2b926118b2, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/e94e4820241f4dfea2d7b51c8544a389, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/03c40a367cce4929b9a61815d9680fb8] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp, totalSize=53.5 K 2024-12-05T22:17:38,275 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 23f95331412a4fca92083b2b926118b2, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733437046137 2024-12-05T22:17:38,276 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting e94e4820241f4dfea2d7b51c8544a389, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733437046150 2024-12-05T22:17:38,276 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 03c40a367cce4929b9a61815d9680fb8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733437056240 2024-12-05T22:17:38,292 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6166a9886eff2aa6d15d170434051ce7#info#compaction#41 average throughput is 12.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:17:38,293 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/74b9b77949e54c4d94fd05342382daf1 is 1080, key is row0001/info:/1733437046137/Put/seqid=0 2024-12-05T22:17:38,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741842_1018 (size=44978) 2024-12-05T22:17:38,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741842_1018 (size=44978) 2024-12-05T22:17:38,304 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/74b9b77949e54c4d94fd05342382daf1 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/74b9b77949e54c4d94fd05342382daf1 2024-12-05T22:17:38,310 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 6166a9886eff2aa6d15d170434051ce7/info of 6166a9886eff2aa6d15d170434051ce7 into 74b9b77949e54c4d94fd05342382daf1(size=43.9 K), total size for store is 43.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:17:38,310 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:38,310 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7., storeName=6166a9886eff2aa6d15d170434051ce7/info, priority=13, startTime=1733437058273; duration=0sec 2024-12-05T22:17:38,310 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=43.9 K, sizeToCheck=16.0 K 2024-12-05T22:17:38,310 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:17:38,310 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/74b9b77949e54c4d94fd05342382daf1 because midkey is the same as first or last row 2024-12-05T22:17:38,311 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:17:38,311 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6166a9886eff2aa6d15d170434051ce7:info 2024-12-05T22:17:38,512 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:39,513 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:39,655 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,656 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,656 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,656 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,656 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,656 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,674 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,674 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,677 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,677 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,677 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:39,679 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,186 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:17:40,187 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,187 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,187 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,187 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,187 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,188 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,213 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,213 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,214 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,214 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,214 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,214 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,217 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,218 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,218 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,220 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:40,513 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:41,514 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:42,515 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:43,515 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:44,486 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:17:44,516 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:45,516 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:46,517 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:47,518 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:47,907 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:17:47,909 INFO [RS-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45580, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:17:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:48,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 6166a9886eff2aa6d15d170434051ce7 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-05T22:17:48,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/ea62bd46624f4592ac0ea928133b9e06 is 1080, key is row0038/info:/1733437058248/Put/seqid=0 2024-12-05T22:17:48,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741843_1019 (size=29761) 2024-12-05T22:17:48,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741843_1019 (size=29761) 2024-12-05T22:17:48,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/ea62bd46624f4592ac0ea928133b9e06 2024-12-05T22:17:48,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/ea62bd46624f4592ac0ea928133b9e06 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/ea62bd46624f4592ac0ea928133b9e06 2024-12-05T22:17:48,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/ea62bd46624f4592ac0ea928133b9e06, entries=23, sequenceid=74, filesize=29.1 K 2024-12-05T22:17:48,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=4.20 KB/4304 for 6166a9886eff2aa6d15d170434051ce7 in 33ms, sequenceid=74, compaction requested=false 2024-12-05T22:17:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=73.0 K, sizeToCheck=16.0 K 2024-12-05T22:17:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:17:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/74b9b77949e54c4d94fd05342382daf1 because midkey is the same as first or last row 2024-12-05T22:17:48,518 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:49,519 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:50,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 6166a9886eff2aa6d15d170434051ce7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:17:50,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/172e1d27d4e6411a806f738c52934afa is 1080, key is row0061/info:/1733437068284/Put/seqid=0 2024-12-05T22:17:50,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741844_1020 (size=12509) 2024-12-05T22:17:50,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741844_1020 (size=12509) 2024-12-05T22:17:50,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/172e1d27d4e6411a806f738c52934afa 2024-12-05T22:17:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/172e1d27d4e6411a806f738c52934afa as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/172e1d27d4e6411a806f738c52934afa 2024-12-05T22:17:50,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T22:17:50,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35300 deadline: 1733437080317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:50,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/172e1d27d4e6411a806f738c52934afa, entries=7, sequenceid=84, filesize=12.2 K 2024-12-05T22:17:50,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 6166a9886eff2aa6d15d170434051ce7 in 26ms, sequenceid=84, compaction requested=true 2024-12-05T22:17:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=85.2 K, sizeToCheck=16.0 K 2024-12-05T22:17:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:17:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/74b9b77949e54c4d94fd05342382daf1 because midkey is the same as first or last row 2024-12-05T22:17:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6166a9886eff2aa6d15d170434051ce7:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:17:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:17:50,320 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:17:50,321 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 87248 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:17:50,321 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1540): 6166a9886eff2aa6d15d170434051ce7/info is initiating minor compaction (all files) 2024-12-05T22:17:50,321 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 6166a9886eff2aa6d15d170434051ce7/info in TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:50,321 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/74b9b77949e54c4d94fd05342382daf1, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/ea62bd46624f4592ac0ea928133b9e06, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/172e1d27d4e6411a806f738c52934afa] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp, totalSize=85.2 K 2024-12-05T22:17:50,322 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 74b9b77949e54c4d94fd05342382daf1, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733437046137 2024-12-05T22:17:50,322 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting ea62bd46624f4592ac0ea928133b9e06, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733437058248 2024-12-05T22:17:50,323 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 172e1d27d4e6411a806f738c52934afa, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733437068284 2024-12-05T22:17:50,346 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6166a9886eff2aa6d15d170434051ce7#info#compaction#44 average throughput is 22.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:17:50,347 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/d772a2e4a82a421b905f019ba7af98f2 is 1080, key is row0001/info:/1733437046137/Put/seqid=0 2024-12-05T22:17:50,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741845_1021 (size=77532) 2024-12-05T22:17:50,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741845_1021 (size=77532) 2024-12-05T22:17:50,355 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/d772a2e4a82a421b905f019ba7af98f2 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/d772a2e4a82a421b905f019ba7af98f2 2024-12-05T22:17:50,360 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 6166a9886eff2aa6d15d170434051ce7/info of 6166a9886eff2aa6d15d170434051ce7 into d772a2e4a82a421b905f019ba7af98f2(size=75.7 K), total size for store is 75.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:17:50,360 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:50,360 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7., storeName=6166a9886eff2aa6d15d170434051ce7/info, priority=13, startTime=1733437070320; duration=0sec 2024-12-05T22:17:50,361 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=75.7 K, sizeToCheck=16.0 K 2024-12-05T22:17:50,361 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-05T22:17:50,362 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:17:50,362 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:17:50,362 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6166a9886eff2aa6d15d170434051ce7:info 2024-12-05T22:17:50,363 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41859 {}] assignment.AssignmentManager(1346): Split request from d029b80fd32e,38777,1733437034635, parent={ENCODED => 6166a9886eff2aa6d15d170434051ce7, NAME => 'TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-12-05T22:17:50,367 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41859 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:50,371 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41859 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=6166a9886eff2aa6d15d170434051ce7, daughterA=dde3fcb9bf2d19e8c2b5cf489cea5b73, daughterB=0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:50,372 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=6166a9886eff2aa6d15d170434051ce7, daughterA=dde3fcb9bf2d19e8c2b5cf489cea5b73, daughterB=0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:50,372 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=6166a9886eff2aa6d15d170434051ce7, daughterA=dde3fcb9bf2d19e8c2b5cf489cea5b73, daughterB=0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:50,372 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=6166a9886eff2aa6d15d170434051ce7, daughterA=dde3fcb9bf2d19e8c2b5cf489cea5b73, daughterB=0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:50,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6166a9886eff2aa6d15d170434051ce7, UNASSIGN}] 2024-12-05T22:17:50,380 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6166a9886eff2aa6d15d170434051ce7, UNASSIGN 2024-12-05T22:17:50,381 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=6166a9886eff2aa6d15d170434051ce7, regionState=CLOSING, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:50,383 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-12-05T22:17:50,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; CloseRegionProcedure 6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635}] 2024-12-05T22:17:50,520 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:50,539 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d029b80fd32e,38777,1733437034635 2024-12-05T22:17:50,541 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(124): Close 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:50,541 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(138): Unassign region: split region: true: evictCache: true 2024-12-05T22:17:50,542 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1681): Closing 6166a9886eff2aa6d15d170434051ce7, disabling compactions & flushes 2024-12-05T22:17:50,542 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:50,542 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:50,542 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. after waiting 0 ms 2024-12-05T22:17:50,542 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:50,542 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(2837): Flushing 6166a9886eff2aa6d15d170434051ce7 1/1 column families, dataSize=23.12 KB heapSize=25 KB 2024-12-05T22:17:50,548 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/4b8289c11b8f48a19ec7eb9df71d0ae8 is 1080, key is row0068/info:/1733437070294/Put/seqid=0 2024-12-05T22:17:50,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741846_1022 (size=28684) 2024-12-05T22:17:50,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741846_1022 (size=28684) 2024-12-05T22:17:50,960 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.12 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/4b8289c11b8f48a19ec7eb9df71d0ae8 2024-12-05T22:17:50,968 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/.tmp/info/4b8289c11b8f48a19ec7eb9df71d0ae8 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/4b8289c11b8f48a19ec7eb9df71d0ae8 2024-12-05T22:17:50,974 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/4b8289c11b8f48a19ec7eb9df71d0ae8, entries=22, sequenceid=110, filesize=28.0 K 2024-12-05T22:17:50,975 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(3040): Finished flush of dataSize ~23.12 KB/23672, heapSize ~24.98 KB/25584, currentSize=0 B/0 for 6166a9886eff2aa6d15d170434051ce7 in 433ms, sequenceid=110, compaction requested=false 2024-12-05T22:17:50,977 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/23f95331412a4fca92083b2b926118b2, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/e94e4820241f4dfea2d7b51c8544a389, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/74b9b77949e54c4d94fd05342382daf1, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/03c40a367cce4929b9a61815d9680fb8, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/ea62bd46624f4592ac0ea928133b9e06, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/172e1d27d4e6411a806f738c52934afa] to archive 2024-12-05T22:17:50,978 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T22:17:50,980 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/23f95331412a4fca92083b2b926118b2 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/23f95331412a4fca92083b2b926118b2 2024-12-05T22:17:50,985 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/e94e4820241f4dfea2d7b51c8544a389 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/e94e4820241f4dfea2d7b51c8544a389 2024-12-05T22:17:50,986 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/74b9b77949e54c4d94fd05342382daf1 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/74b9b77949e54c4d94fd05342382daf1 2024-12-05T22:17:50,988 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/03c40a367cce4929b9a61815d9680fb8 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/03c40a367cce4929b9a61815d9680fb8 2024-12-05T22:17:50,989 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/ea62bd46624f4592ac0ea928133b9e06 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/ea62bd46624f4592ac0ea928133b9e06 2024-12-05T22:17:50,990 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/172e1d27d4e6411a806f738c52934afa to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/172e1d27d4e6411a806f738c52934afa 2024-12-05T22:17:50,999 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/recovered.edits/113.seqid, newMaxSeqId=113, maxSeqId=1 2024-12-05T22:17:50,999 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. 2024-12-05T22:17:51,000 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1635): Region close journal for 6166a9886eff2aa6d15d170434051ce7: 2024-12-05T22:17:51,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=6166a9886eff2aa6d15d170434051ce7, regionState=CLOSED 2024-12-05T22:17:51,003 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(170): Closed 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:51,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T22:17:51,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; CloseRegionProcedure 6166a9886eff2aa6d15d170434051ce7, server=d029b80fd32e,38777,1733437034635 in 624 msec 2024-12-05T22:17:51,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-12-05T22:17:51,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6166a9886eff2aa6d15d170434051ce7, UNASSIGN in 631 msec 2024-12-05T22:17:51,049 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:51,050 INFO [PEWorker-1 {}] assignment.SplitTableRegionProcedure(728): pid=12 splitting 2 storefiles, region=6166a9886eff2aa6d15d170434051ce7, threads=2 2024-12-05T22:17:51,051 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/4b8289c11b8f48a19ec7eb9df71d0ae8 for region: 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:51,056 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/d772a2e4a82a421b905f019ba7af98f2 for region: 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:51,070 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/4b8289c11b8f48a19ec7eb9df71d0ae8, top=true 2024-12-05T22:17:51,098 INFO [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/TestLogRolling-testLogRolling=6166a9886eff2aa6d15d170434051ce7-4b8289c11b8f48a19ec7eb9df71d0ae8 for child: 0088fb5c8612929b4093695e13cbc0fc, parent: 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:51,098 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/4b8289c11b8f48a19ec7eb9df71d0ae8 for region: 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:51,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741847_1023 (size=27) 2024-12-05T22:17:51,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741847_1023 (size=27) 2024-12-05T22:17:51,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741848_1024 (size=27) 2024-12-05T22:17:51,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741848_1024 (size=27) 2024-12-05T22:17:51,153 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/d772a2e4a82a421b905f019ba7af98f2 for region: 6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:17:51,153 DEBUG [PEWorker-1 {}] assignment.SplitTableRegionProcedure(802): pid=12 split storefiles for region 6166a9886eff2aa6d15d170434051ce7 Daughter A: [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7] storefiles, Daughter B: [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/TestLogRolling-testLogRolling=6166a9886eff2aa6d15d170434051ce7-4b8289c11b8f48a19ec7eb9df71d0ae8, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7] storefiles. 2024-12-05T22:17:51,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741849_1025 (size=71) 2024-12-05T22:17:51,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741849_1025 (size=71) 2024-12-05T22:17:51,174 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:51,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741850_1026 (size=71) 2024-12-05T22:17:51,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741850_1026 (size=71) 2024-12-05T22:17:51,192 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:51,205 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/recovered.edits/113.seqid, newMaxSeqId=113, maxSeqId=-1 2024-12-05T22:17:51,210 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/recovered.edits/113.seqid, newMaxSeqId=113, maxSeqId=-1 2024-12-05T22:17:51,214 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733437071214"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733437071214"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733437071214"}]},"ts":"1733437071214"} 2024-12-05T22:17:51,214 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733437071214"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733437071214"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733437071214"}]},"ts":"1733437071214"} 2024-12-05T22:17:51,214 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733437071214"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733437071214"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733437071214"}]},"ts":"1733437071214"} 2024-12-05T22:17:51,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38777 {}] regionserver.HRegion(8581): Flush requested on 1588230740 2024-12-05T22:17:51,253 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T22:17:51,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=4.75 KB heapSize=8.29 KB 2024-12-05T22:17:51,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=dde3fcb9bf2d19e8c2b5cf489cea5b73, ASSIGN}, {pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0088fb5c8612929b4093695e13cbc0fc, ASSIGN}] 2024-12-05T22:17:51,268 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=dde3fcb9bf2d19e8c2b5cf489cea5b73, ASSIGN 2024-12-05T22:17:51,268 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0088fb5c8612929b4093695e13cbc0fc, ASSIGN 2024-12-05T22:17:51,269 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=dde3fcb9bf2d19e8c2b5cf489cea5b73, ASSIGN; state=SPLITTING_NEW, location=d029b80fd32e,38777,1733437034635; forceNewPlan=false, retain=false 2024-12-05T22:17:51,269 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0088fb5c8612929b4093695e13cbc0fc, ASSIGN; state=SPLITTING_NEW, location=d029b80fd32e,38777,1733437034635; forceNewPlan=false, retain=false 2024-12-05T22:17:51,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/.tmp/info/245b96a38a064600aaa18fba732dce85 is 193, key is TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc./info:regioninfo/1733437071214/Put/seqid=0 2024-12-05T22:17:51,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741851_1027 (size=9423) 2024-12-05T22:17:51,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741851_1027 (size=9423) 2024-12-05T22:17:51,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.54 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/.tmp/info/245b96a38a064600aaa18fba732dce85 2024-12-05T22:17:51,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/.tmp/table/e763f8a5884c40668187b9da014c8488 is 65, key is TestLogRolling-testLogRolling/table:state/1733437036914/Put/seqid=0 2024-12-05T22:17:51,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741852_1028 (size=5412) 2024-12-05T22:17:51,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741852_1028 (size=5412) 2024-12-05T22:17:51,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=216 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/.tmp/table/e763f8a5884c40668187b9da014c8488 2024-12-05T22:17:51,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/.tmp/info/245b96a38a064600aaa18fba732dce85 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/info/245b96a38a064600aaa18fba732dce85 2024-12-05T22:17:51,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/info/245b96a38a064600aaa18fba732dce85, entries=29, sequenceid=17, filesize=9.2 K 2024-12-05T22:17:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/.tmp/table/e763f8a5884c40668187b9da014c8488 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/table/e763f8a5884c40668187b9da014c8488 2024-12-05T22:17:51,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/table/e763f8a5884c40668187b9da014c8488, entries=4, sequenceid=17, filesize=5.3 K 2024-12-05T22:17:51,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~4.75 KB/4869, heapSize ~8.01 KB/8200, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=17, compaction requested=false 2024-12-05T22:17:51,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-12-05T22:17:51,420 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=dde3fcb9bf2d19e8c2b5cf489cea5b73, regionState=OPENING, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:51,420 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=0088fb5c8612929b4093695e13cbc0fc, regionState=OPENING, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:51,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=15, state=RUNNABLE; OpenRegionProcedure dde3fcb9bf2d19e8c2b5cf489cea5b73, server=d029b80fd32e,38777,1733437034635}] 2024-12-05T22:17:51,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=16, state=RUNNABLE; OpenRegionProcedure 0088fb5c8612929b4093695e13cbc0fc, server=d029b80fd32e,38777,1733437034635}] 2024-12-05T22:17:51,520 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:51,574 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d029b80fd32e,38777,1733437034635 2024-12-05T22:17:51,578 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:17:51,579 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => dde3fcb9bf2d19e8c2b5cf489cea5b73, NAME => 'TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73.', STARTKEY => '', ENDKEY => 'row0062'} 2024-12-05T22:17:51,579 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling dde3fcb9bf2d19e8c2b5cf489cea5b73 2024-12-05T22:17:51,579 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:51,579 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for dde3fcb9bf2d19e8c2b5cf489cea5b73 2024-12-05T22:17:51,579 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for dde3fcb9bf2d19e8c2b5cf489cea5b73 2024-12-05T22:17:51,581 INFO [StoreOpener-dde3fcb9bf2d19e8c2b5cf489cea5b73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region dde3fcb9bf2d19e8c2b5cf489cea5b73 2024-12-05T22:17:51,582 INFO [StoreOpener-dde3fcb9bf2d19e8c2b5cf489cea5b73-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde3fcb9bf2d19e8c2b5cf489cea5b73 columnFamilyName info 2024-12-05T22:17:51,582 DEBUG [StoreOpener-dde3fcb9bf2d19e8c2b5cf489cea5b73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:51,599 DEBUG [StoreOpener-dde3fcb9bf2d19e8c2b5cf489cea5b73-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7->hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/d772a2e4a82a421b905f019ba7af98f2-bottom 2024-12-05T22:17:51,599 INFO [StoreOpener-dde3fcb9bf2d19e8c2b5cf489cea5b73-1 {}] regionserver.HStore(327): Store=dde3fcb9bf2d19e8c2b5cf489cea5b73/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:17:51,600 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73 2024-12-05T22:17:51,602 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73 2024-12-05T22:17:51,604 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for dde3fcb9bf2d19e8c2b5cf489cea5b73 2024-12-05T22:17:51,605 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1102): Opened dde3fcb9bf2d19e8c2b5cf489cea5b73; next sequenceid=114; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=818004, jitterRate=0.04014602303504944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:17:51,606 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for dde3fcb9bf2d19e8c2b5cf489cea5b73: 2024-12-05T22:17:51,607 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73., pid=17, masterSystemTime=1733437071574 2024-12-05T22:17:51,607 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(403): Add compact mark for store dde3fcb9bf2d19e8c2b5cf489cea5b73:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:17:51,607 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:17:51,607 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-05T22:17:51,608 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:17:51,608 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1540): dde3fcb9bf2d19e8c2b5cf489cea5b73/info is initiating minor compaction (all files) 2024-12-05T22:17:51,608 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dde3fcb9bf2d19e8c2b5cf489cea5b73/info in TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:17:51,608 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7->hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/d772a2e4a82a421b905f019ba7af98f2-bottom] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/.tmp, totalSize=75.7 K 2024-12-05T22:17:51,609 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7, keycount=33, bloomtype=ROW, size=75.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733437046137 2024-12-05T22:17:51,609 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:17:51,610 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:17:51,610 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:17:51,610 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=dde3fcb9bf2d19e8c2b5cf489cea5b73, regionState=OPEN, openSeqNum=114, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:51,610 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7285): Opening region: {ENCODED => 0088fb5c8612929b4093695e13cbc0fc, NAME => 'TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.', STARTKEY => 'row0062', ENDKEY => ''} 2024-12-05T22:17:51,610 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:51,610 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:17:51,610 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7327): checking encryption for 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:51,610 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7330): checking classloading for 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:51,612 INFO [StoreOpener-0088fb5c8612929b4093695e13cbc0fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:51,613 INFO [StoreOpener-0088fb5c8612929b4093695e13cbc0fc-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0088fb5c8612929b4093695e13cbc0fc columnFamilyName info 2024-12-05T22:17:51,613 DEBUG [StoreOpener-0088fb5c8612929b4093695e13cbc0fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:17:51,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=15 2024-12-05T22:17:51,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=15, state=SUCCESS; OpenRegionProcedure dde3fcb9bf2d19e8c2b5cf489cea5b73, server=d029b80fd32e,38777,1733437034635 in 190 msec 2024-12-05T22:17:51,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=dde3fcb9bf2d19e8c2b5cf489cea5b73, ASSIGN in 350 msec 2024-12-05T22:17:51,630 DEBUG [StoreOpener-0088fb5c8612929b4093695e13cbc0fc-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/TestLogRolling-testLogRolling=6166a9886eff2aa6d15d170434051ce7-4b8289c11b8f48a19ec7eb9df71d0ae8 2024-12-05T22:17:51,635 DEBUG [StoreOpener-0088fb5c8612929b4093695e13cbc0fc-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7->hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/d772a2e4a82a421b905f019ba7af98f2-top 2024-12-05T22:17:51,635 INFO [StoreOpener-0088fb5c8612929b4093695e13cbc0fc-1 {}] regionserver.HStore(327): Store=0088fb5c8612929b4093695e13cbc0fc/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:17:51,636 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:51,637 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:51,638 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dde3fcb9bf2d19e8c2b5cf489cea5b73#info#compaction#48 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:17:51,638 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/.tmp/info/350314fdc3dc47c58cad27cab9495b76 is 1080, key is row0001/info:/1733437046137/Put/seqid=0 2024-12-05T22:17:51,642 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1085): writing seq id for 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:17:51,643 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1102): Opened 0088fb5c8612929b4093695e13cbc0fc; next sequenceid=114; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=749730, jitterRate=-0.0466691255569458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:17:51,643 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1001): Region open journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:17:51,644 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc., pid=18, masterSystemTime=1733437071574 2024-12-05T22:17:51,645 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(403): Add compact mark for store 0088fb5c8612929b4093695e13cbc0fc:info, priority=-2147483648, current under compaction store size is 2 2024-12-05T22:17:51,645 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:17:51,645 DEBUG [RS:0;d029b80fd32e:38777-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T22:17:51,646 INFO [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:17:51,646 DEBUG [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.HStore(1540): 0088fb5c8612929b4093695e13cbc0fc/info is initiating minor compaction (all files) 2024-12-05T22:17:51,646 INFO [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0088fb5c8612929b4093695e13cbc0fc/info in TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:17:51,646 INFO [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7->hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/d772a2e4a82a421b905f019ba7af98f2-top, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/TestLogRolling-testLogRolling=6166a9886eff2aa6d15d170434051ce7-4b8289c11b8f48a19ec7eb9df71d0ae8] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp, totalSize=103.7 K 2024-12-05T22:17:51,647 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=0088fb5c8612929b4093695e13cbc0fc, regionState=OPEN, openSeqNum=114, regionLocation=d029b80fd32e,38777,1733437034635 2024-12-05T22:17:51,648 DEBUG [RS:0;d029b80fd32e:38777-longCompactions-0 {}] compactions.Compactor(224): Compacting d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7, keycount=33, bloomtype=ROW, size=75.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733437046137 2024-12-05T22:17:51,650 DEBUG [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:17:51,651 INFO [RS_OPEN_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:17:51,651 DEBUG [RS:0;d029b80fd32e:38777-longCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=6166a9886eff2aa6d15d170434051ce7-4b8289c11b8f48a19ec7eb9df71d0ae8, keycount=22, bloomtype=ROW, size=28.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733437070294 2024-12-05T22:17:51,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=16 2024-12-05T22:17:51,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=16, state=SUCCESS; OpenRegionProcedure 0088fb5c8612929b4093695e13cbc0fc, server=d029b80fd32e,38777,1733437034635 in 228 msec 2024-12-05T22:17:51,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=12 2024-12-05T22:17:51,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0088fb5c8612929b4093695e13cbc0fc, ASSIGN in 388 msec 2024-12-05T22:17:51,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=6166a9886eff2aa6d15d170434051ce7, daughterA=dde3fcb9bf2d19e8c2b5cf489cea5b73, daughterB=0088fb5c8612929b4093695e13cbc0fc in 1.2880 sec 2024-12-05T22:17:51,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741853_1029 (size=70862) 2024-12-05T22:17:51,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741853_1029 (size=70862) 2024-12-05T22:17:51,687 INFO [RS:0;d029b80fd32e:38777-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0088fb5c8612929b4093695e13cbc0fc#info#compaction#49 average throughput is 14.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:17:51,688 DEBUG [RS:0;d029b80fd32e:38777-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/d2e1ab2700984a3c87f94431621426de is 1080, key is row0062/info:/1733437068286/Put/seqid=0 2024-12-05T22:17:51,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741854_1030 (size=35344) 2024-12-05T22:17:51,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741854_1030 (size=35344) 2024-12-05T22:17:51,713 DEBUG [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/d2e1ab2700984a3c87f94431621426de as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d2e1ab2700984a3c87f94431621426de 2024-12-05T22:17:51,722 INFO [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 2 (all) file(s) in 0088fb5c8612929b4093695e13cbc0fc/info of 0088fb5c8612929b4093695e13cbc0fc into d2e1ab2700984a3c87f94431621426de(size=34.5 K), total size for store is 34.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:17:51,722 DEBUG [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:17:51,722 INFO [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc., storeName=0088fb5c8612929b4093695e13cbc0fc/info, priority=14, startTime=1733437071645; duration=0sec 2024-12-05T22:17:51,722 DEBUG [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:17:51,722 DEBUG [RS:0;d029b80fd32e:38777-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0088fb5c8612929b4093695e13cbc0fc:info 2024-12-05T22:17:52,071 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/.tmp/info/350314fdc3dc47c58cad27cab9495b76 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/info/350314fdc3dc47c58cad27cab9495b76 2024-12-05T22:17:52,078 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 1 (all) file(s) in dde3fcb9bf2d19e8c2b5cf489cea5b73/info of dde3fcb9bf2d19e8c2b5cf489cea5b73 into 350314fdc3dc47c58cad27cab9495b76(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:17:52,078 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dde3fcb9bf2d19e8c2b5cf489cea5b73: 2024-12-05T22:17:52,078 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73., storeName=dde3fcb9bf2d19e8c2b5cf489cea5b73/info, priority=15, startTime=1733437071607; duration=0sec 2024-12-05T22:17:52,079 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:17:52,079 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dde3fcb9bf2d19e8c2b5cf489cea5b73:info 2024-12-05T22:17:52,521 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:53,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:54,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:55,523 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:56,000 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,000 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,001 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,001 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,001 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,001 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,030 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,030 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,030 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,030 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,031 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,031 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,035 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,036 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,036 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,039 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,063 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T22:17:56,063 INFO [master/d029b80fd32e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T22:17:56,524 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:56,550 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-05T22:17:56,551 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,551 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,551 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,551 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,552 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,552 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,569 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,569 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,569 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,570 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,570 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,570 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,573 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,573 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,573 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:56,575 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-05T22:17:57,524 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:58,525 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:17:59,526 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35300 deadline: 1733437090377, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733437036123.6166a9886eff2aa6d15d170434051ce7. is not online on d029b80fd32e,38777,1733437034635 2024-12-05T22:18:00,502 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835 2024-12-05T22:18:00,526 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:01,527 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:02,527 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:03,528 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:04,529 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:05,530 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:06,530 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:07,531 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:08,531 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:09,532 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:10,533 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:11,533 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:12,534 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:13,535 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:14,487 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:18:14,535 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:15,536 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:15,581 DEBUG [master/d029b80fd32e:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 883cc4622e8d8175d333f4fa3e2982b4 changed from -1.0 to 0.0, refreshing cache 2024-12-05T22:18:16,536 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:17,537 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:18,538 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:19,538 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:20,539 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:20,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:18:20,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/98dba5760692407ea45e7c66425b5253 is 1080, key is row0090/info:/1733437100578/Put/seqid=0 2024-12-05T22:18:20,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741855_1031 (size=12509) 2024-12-05T22:18:20,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741855_1031 (size=12509) 2024-12-05T22:18:20,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/98dba5760692407ea45e7c66425b5253 2024-12-05T22:18:20,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/98dba5760692407ea45e7c66425b5253 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/98dba5760692407ea45e7c66425b5253 2024-12-05T22:18:20,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/98dba5760692407ea45e7c66425b5253, entries=7, sequenceid=124, filesize=12.2 K 2024-12-05T22:18:20,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 0088fb5c8612929b4093695e13cbc0fc in 47ms, sequenceid=124, compaction requested=false 2024-12-05T22:18:20,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:21,539 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:22,540 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:22,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:18:22,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/ea540428fa1a490d8588d87653420dad is 1080, key is row0097/info:/1733437102588/Put/seqid=0 2024-12-05T22:18:22,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741856_1032 (size=12515) 2024-12-05T22:18:22,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741856_1032 (size=12515) 2024-12-05T22:18:22,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/ea540428fa1a490d8588d87653420dad 2024-12-05T22:18:22,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=0088fb5c8612929b4093695e13cbc0fc, server=d029b80fd32e,38777,1733437034635 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T22:18:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35300 deadline: 1733437112622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=0088fb5c8612929b4093695e13cbc0fc, server=d029b80fd32e,38777,1733437034635 2024-12-05T22:18:22,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/ea540428fa1a490d8588d87653420dad as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/ea540428fa1a490d8588d87653420dad 2024-12-05T22:18:22,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/ea540428fa1a490d8588d87653420dad, entries=7, sequenceid=134, filesize=12.2 K 2024-12-05T22:18:22,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 0088fb5c8612929b4093695e13cbc0fc in 38ms, sequenceid=134, compaction requested=true 2024-12-05T22:18:22,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:22,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0088fb5c8612929b4093695e13cbc0fc:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:18:22,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:22,636 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:18:22,637 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 60368 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:18:22,637 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1540): 0088fb5c8612929b4093695e13cbc0fc/info is initiating minor compaction (all files) 2024-12-05T22:18:22,637 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0088fb5c8612929b4093695e13cbc0fc/info in TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:18:22,637 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d2e1ab2700984a3c87f94431621426de, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/98dba5760692407ea45e7c66425b5253, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/ea540428fa1a490d8588d87653420dad] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp, totalSize=59.0 K 2024-12-05T22:18:22,638 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting d2e1ab2700984a3c87f94431621426de, keycount=28, bloomtype=ROW, size=34.5 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733437068286 2024-12-05T22:18:22,638 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 98dba5760692407ea45e7c66425b5253, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733437100578 2024-12-05T22:18:22,639 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting ea540428fa1a490d8588d87653420dad, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733437102588 2024-12-05T22:18:22,657 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0088fb5c8612929b4093695e13cbc0fc#info#compaction#52 average throughput is 43.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:18:22,657 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/b8066221ed5843bcbadd71c85fc115c4 is 1080, key is row0062/info:/1733437068286/Put/seqid=0 2024-12-05T22:18:22,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741857_1033 (size=50534) 2024-12-05T22:18:22,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741857_1033 (size=50534) 2024-12-05T22:18:22,668 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/b8066221ed5843bcbadd71c85fc115c4 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/b8066221ed5843bcbadd71c85fc115c4 2024-12-05T22:18:22,673 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0088fb5c8612929b4093695e13cbc0fc/info of 0088fb5c8612929b4093695e13cbc0fc into b8066221ed5843bcbadd71c85fc115c4(size=49.3 K), total size for store is 49.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:18:22,673 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:22,673 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc., storeName=0088fb5c8612929b4093695e13cbc0fc/info, priority=13, startTime=1733437102636; duration=0sec 2024-12-05T22:18:22,674 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:22,674 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0088fb5c8612929b4093695e13cbc0fc:info 2024-12-05T22:18:23,541 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:24,541 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:25,542 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:26,542 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:27,543 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:28,544 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:28,545 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=3 on file=hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta after 196130ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor199.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T22:18:29,545 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:30,546 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:31,546 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:32,547 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:32,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-05T22:18:32,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/0183a54d69594f3cadb95fb31c4d7cfb is 1080, key is row0104/info:/1733437102598/Put/seqid=0 2024-12-05T22:18:32,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741858_1034 (size=29784) 2024-12-05T22:18:32,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741858_1034 (size=29784) 2024-12-05T22:18:32,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/0183a54d69594f3cadb95fb31c4d7cfb 2024-12-05T22:18:32,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/0183a54d69594f3cadb95fb31c4d7cfb as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/0183a54d69594f3cadb95fb31c4d7cfb 2024-12-05T22:18:32,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/0183a54d69594f3cadb95fb31c4d7cfb, entries=23, sequenceid=161, filesize=29.1 K 2024-12-05T22:18:32,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 0088fb5c8612929b4093695e13cbc0fc in 38ms, sequenceid=161, compaction requested=false 2024-12-05T22:18:32,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:33,548 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:34,548 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:34,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:18:34,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/8b158fb989d447e7a9fe7ee6dddfc490 is 1080, key is row0127/info:/1733437112716/Put/seqid=0 2024-12-05T22:18:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741859_1035 (size=12516) 2024-12-05T22:18:34,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741859_1035 (size=12516) 2024-12-05T22:18:34,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/8b158fb989d447e7a9fe7ee6dddfc490 2024-12-05T22:18:34,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/8b158fb989d447e7a9fe7ee6dddfc490 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/8b158fb989d447e7a9fe7ee6dddfc490 2024-12-05T22:18:34,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/8b158fb989d447e7a9fe7ee6dddfc490, entries=7, sequenceid=171, filesize=12.2 K 2024-12-05T22:18:34,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=22.07 KB/22596 for 0088fb5c8612929b4093695e13cbc0fc in 23ms, sequenceid=171, compaction requested=true 2024-12-05T22:18:34,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:34,750 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:18:34,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0088fb5c8612929b4093695e13cbc0fc:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:18:34,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:34,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=23.12 KB heapSize=25 KB 2024-12-05T22:18:34,751 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92834 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:18:34,751 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1540): 0088fb5c8612929b4093695e13cbc0fc/info is initiating minor compaction (all files) 2024-12-05T22:18:34,752 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0088fb5c8612929b4093695e13cbc0fc/info in TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:18:34,752 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/b8066221ed5843bcbadd71c85fc115c4, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/0183a54d69594f3cadb95fb31c4d7cfb, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/8b158fb989d447e7a9fe7ee6dddfc490] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp, totalSize=90.7 K 2024-12-05T22:18:34,752 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting b8066221ed5843bcbadd71c85fc115c4, keycount=42, bloomtype=ROW, size=49.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733437068286 2024-12-05T22:18:34,753 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0183a54d69594f3cadb95fb31c4d7cfb, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733437102598 2024-12-05T22:18:34,753 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8b158fb989d447e7a9fe7ee6dddfc490, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733437112716 2024-12-05T22:18:34,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/721a9b251e1840a889ef14b415a491be is 1080, key is row0134/info:/1733437114727/Put/seqid=0 2024-12-05T22:18:34,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741860_1036 (size=28706) 2024-12-05T22:18:34,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741860_1036 (size=28706) 2024-12-05T22:18:34,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.12 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/721a9b251e1840a889ef14b415a491be 2024-12-05T22:18:34,767 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0088fb5c8612929b4093695e13cbc0fc#info#compaction#56 average throughput is 36.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:18:34,768 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/035de0c4e74c4fb994b1c02725ff56c0 is 1080, key is row0062/info:/1733437068286/Put/seqid=0 2024-12-05T22:18:34,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/721a9b251e1840a889ef14b415a491be as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/721a9b251e1840a889ef14b415a491be 2024-12-05T22:18:34,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/721a9b251e1840a889ef14b415a491be, entries=22, sequenceid=196, filesize=28.0 K 2024-12-05T22:18:34,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~23.12 KB/23672, heapSize ~24.98 KB/25584, currentSize=5.25 KB/5380 for 0088fb5c8612929b4093695e13cbc0fc in 25ms, sequenceid=196, compaction requested=false 2024-12-05T22:18:34,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:34,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741861_1037 (size=83117) 2024-12-05T22:18:34,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741861_1037 (size=83117) 2024-12-05T22:18:34,783 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/035de0c4e74c4fb994b1c02725ff56c0 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/035de0c4e74c4fb994b1c02725ff56c0 2024-12-05T22:18:34,789 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0088fb5c8612929b4093695e13cbc0fc/info of 0088fb5c8612929b4093695e13cbc0fc into 035de0c4e74c4fb994b1c02725ff56c0(size=81.2 K), total size for store is 109.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:18:34,789 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:34,789 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc., storeName=0088fb5c8612929b4093695e13cbc0fc/info, priority=13, startTime=1733437114750; duration=0sec 2024-12-05T22:18:34,789 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:34,789 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0088fb5c8612929b4093695e13cbc0fc:info 2024-12-05T22:18:35,549 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:36,549 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:36,579 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region dde3fcb9bf2d19e8c2b5cf489cea5b73, had cached 0 bytes from a total of 70862 2024-12-05T22:18:36,610 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 0088fb5c8612929b4093695e13cbc0fc, had cached 0 bytes from a total of 111823 2024-12-05T22:18:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:36,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:18:36,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/56c1fb0178b64bf7b4029fe4ceb5bc46 is 1080, key is row0156/info:/1733437114752/Put/seqid=0 2024-12-05T22:18:36,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741862_1038 (size=12516) 2024-12-05T22:18:36,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741862_1038 (size=12516) 2024-12-05T22:18:36,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/56c1fb0178b64bf7b4029fe4ceb5bc46 2024-12-05T22:18:36,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/56c1fb0178b64bf7b4029fe4ceb5bc46 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/56c1fb0178b64bf7b4029fe4ceb5bc46 2024-12-05T22:18:36,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/56c1fb0178b64bf7b4029fe4ceb5bc46, entries=7, sequenceid=207, filesize=12.2 K 2024-12-05T22:18:36,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 0088fb5c8612929b4093695e13cbc0fc in 31ms, sequenceid=207, compaction requested=true 2024-12-05T22:18:36,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:36,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0088fb5c8612929b4093695e13cbc0fc:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:18:36,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:36,793 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:18:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:36,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-05T22:18:36,794 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 124339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:18:36,794 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1540): 0088fb5c8612929b4093695e13cbc0fc/info is initiating minor compaction (all files) 2024-12-05T22:18:36,794 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0088fb5c8612929b4093695e13cbc0fc/info in TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:18:36,794 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/035de0c4e74c4fb994b1c02725ff56c0, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/721a9b251e1840a889ef14b415a491be, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/56c1fb0178b64bf7b4029fe4ceb5bc46] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp, totalSize=121.4 K 2024-12-05T22:18:36,795 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 035de0c4e74c4fb994b1c02725ff56c0, keycount=72, bloomtype=ROW, size=81.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733437068286 2024-12-05T22:18:36,795 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 721a9b251e1840a889ef14b415a491be, keycount=22, bloomtype=ROW, size=28.0 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733437114727 2024-12-05T22:18:36,796 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 56c1fb0178b64bf7b4029fe4ceb5bc46, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733437114752 2024-12-05T22:18:36,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/3496028516274db6ba57ee71bd3b2c73 is 1080, key is row0163/info:/1733437116763/Put/seqid=0 2024-12-05T22:18:36,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=0088fb5c8612929b4093695e13cbc0fc, server=d029b80fd32e,38777,1733437034635 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T22:18:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35300 deadline: 1733437126800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=0088fb5c8612929b4093695e13cbc0fc, server=d029b80fd32e,38777,1733437034635 2024-12-05T22:18:36,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741863_1039 (size=29784) 2024-12-05T22:18:36,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741863_1039 (size=29784) 2024-12-05T22:18:36,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/3496028516274db6ba57ee71bd3b2c73 2024-12-05T22:18:36,808 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0088fb5c8612929b4093695e13cbc0fc#info#compaction#59 average throughput is 34.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:18:36,809 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/90e792416c164c0198e49737f6f81a8f is 1080, key is row0062/info:/1733437068286/Put/seqid=0 2024-12-05T22:18:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/3496028516274db6ba57ee71bd3b2c73 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/3496028516274db6ba57ee71bd3b2c73 2024-12-05T22:18:36,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741864_1040 (size=114489) 2024-12-05T22:18:36,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741864_1040 (size=114489) 2024-12-05T22:18:36,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/3496028516274db6ba57ee71bd3b2c73, entries=23, sequenceid=233, filesize=29.1 K 2024-12-05T22:18:36,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=6.30 KB/6456 for 0088fb5c8612929b4093695e13cbc0fc in 25ms, sequenceid=233, compaction requested=false 2024-12-05T22:18:36,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:36,820 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/90e792416c164c0198e49737f6f81a8f as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/90e792416c164c0198e49737f6f81a8f 2024-12-05T22:18:36,826 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0088fb5c8612929b4093695e13cbc0fc/info of 0088fb5c8612929b4093695e13cbc0fc into 90e792416c164c0198e49737f6f81a8f(size=111.8 K), total size for store is 140.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:18:36,826 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:36,826 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc., storeName=0088fb5c8612929b4093695e13cbc0fc/info, priority=13, startTime=1733437116793; duration=0sec 2024-12-05T22:18:36,826 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:36,826 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0088fb5c8612929b4093695e13cbc0fc:info 2024-12-05T22:18:37,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:38,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:39,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:40,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:41,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:42,553 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:43,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:44,487 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T22:18:44,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:45,502 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835 2024-12-05T22:18:45,555 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:46,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:46,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:18:46,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/d0b2054b5eeb4b90936588de90eb3a58 is 1080, key is row0186/info:/1733437116794/Put/seqid=0 2024-12-05T22:18:46,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741865_1041 (size=12516) 2024-12-05T22:18:46,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741865_1041 (size=12516) 2024-12-05T22:18:46,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/d0b2054b5eeb4b90936588de90eb3a58 2024-12-05T22:18:46,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/d0b2054b5eeb4b90936588de90eb3a58 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d0b2054b5eeb4b90936588de90eb3a58 2024-12-05T22:18:46,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d0b2054b5eeb4b90936588de90eb3a58, entries=7, sequenceid=244, filesize=12.2 K 2024-12-05T22:18:46,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 0088fb5c8612929b4093695e13cbc0fc in 30ms, sequenceid=244, compaction requested=true 2024-12-05T22:18:46,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:46,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0088fb5c8612929b4093695e13cbc0fc:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:18:46,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:46,857 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:18:46,858 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 156789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:18:46,858 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1540): 0088fb5c8612929b4093695e13cbc0fc/info is initiating minor compaction (all files) 2024-12-05T22:18:46,858 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0088fb5c8612929b4093695e13cbc0fc/info in TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:18:46,858 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/90e792416c164c0198e49737f6f81a8f, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/3496028516274db6ba57ee71bd3b2c73, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d0b2054b5eeb4b90936588de90eb3a58] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp, totalSize=153.1 K 2024-12-05T22:18:46,859 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 90e792416c164c0198e49737f6f81a8f, keycount=101, bloomtype=ROW, size=111.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733437068286 2024-12-05T22:18:46,859 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3496028516274db6ba57ee71bd3b2c73, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733437116763 2024-12-05T22:18:46,859 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting d0b2054b5eeb4b90936588de90eb3a58, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733437116794 2024-12-05T22:18:46,871 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0088fb5c8612929b4093695e13cbc0fc#info#compaction#61 average throughput is 67.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:18:46,871 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/68c42d4a5c55442d8f45538d7779dded is 1080, key is row0062/info:/1733437068286/Put/seqid=0 2024-12-05T22:18:46,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741866_1042 (size=147136) 2024-12-05T22:18:46,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741866_1042 (size=147136) 2024-12-05T22:18:46,884 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/68c42d4a5c55442d8f45538d7779dded as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/68c42d4a5c55442d8f45538d7779dded 2024-12-05T22:18:46,891 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0088fb5c8612929b4093695e13cbc0fc/info of 0088fb5c8612929b4093695e13cbc0fc into 68c42d4a5c55442d8f45538d7779dded(size=143.7 K), total size for store is 143.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:18:46,891 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:46,891 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc., storeName=0088fb5c8612929b4093695e13cbc0fc/info, priority=13, startTime=1733437126857; duration=0sec 2024-12-05T22:18:46,891 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:46,891 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0088fb5c8612929b4093695e13cbc0fc:info 2024-12-05T22:18:47,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:48,557 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:48,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:18:48,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/936b30b8b6164685b36998231cbe6f55 is 1080, key is row0193/info:/1733437128828/Put/seqid=0 2024-12-05T22:18:48,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741867_1043 (size=12516) 2024-12-05T22:18:48,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741867_1043 (size=12516) 2024-12-05T22:18:48,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/936b30b8b6164685b36998231cbe6f55 2024-12-05T22:18:48,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/936b30b8b6164685b36998231cbe6f55 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/936b30b8b6164685b36998231cbe6f55 2024-12-05T22:18:48,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=0088fb5c8612929b4093695e13cbc0fc, server=d029b80fd32e,38777,1733437034635 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T22:18:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:35300 deadline: 1733437138869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=0088fb5c8612929b4093695e13cbc0fc, server=d029b80fd32e,38777,1733437034635 2024-12-05T22:18:48,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/936b30b8b6164685b36998231cbe6f55, entries=7, sequenceid=255, filesize=12.2 K 2024-12-05T22:18:48,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 0088fb5c8612929b4093695e13cbc0fc in 35ms, sequenceid=255, compaction requested=false 2024-12-05T22:18:48,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:49,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:50,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:51,559 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:52,559 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:53,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:54,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:55,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:56,562 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:57,562 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:58,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:18:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:18:58,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-12-05T22:18:58,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/67ff5b7806924fe7a5265882eeb025eb is 1080, key is row0200/info:/1733437128836/Put/seqid=0 2024-12-05T22:18:58,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741868_1044 (size=29807) 2024-12-05T22:18:58,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741868_1044 (size=29807) 2024-12-05T22:18:58,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/67ff5b7806924fe7a5265882eeb025eb 2024-12-05T22:18:58,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/67ff5b7806924fe7a5265882eeb025eb as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/67ff5b7806924fe7a5265882eeb025eb 2024-12-05T22:18:58,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/67ff5b7806924fe7a5265882eeb025eb, entries=23, sequenceid=281, filesize=29.1 K 2024-12-05T22:18:58,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 0088fb5c8612929b4093695e13cbc0fc in 22ms, sequenceid=281, compaction requested=true 2024-12-05T22:18:58,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:58,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0088fb5c8612929b4093695e13cbc0fc:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:18:58,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:58,946 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:18:58,947 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 189459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:18:58,947 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1540): 0088fb5c8612929b4093695e13cbc0fc/info is initiating minor compaction (all files) 2024-12-05T22:18:58,947 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0088fb5c8612929b4093695e13cbc0fc/info in TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:18:58,947 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/68c42d4a5c55442d8f45538d7779dded, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/936b30b8b6164685b36998231cbe6f55, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/67ff5b7806924fe7a5265882eeb025eb] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp, totalSize=185.0 K 2024-12-05T22:18:58,947 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 68c42d4a5c55442d8f45538d7779dded, keycount=131, bloomtype=ROW, size=143.7 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733437068286 2024-12-05T22:18:58,947 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 936b30b8b6164685b36998231cbe6f55, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733437128828 2024-12-05T22:18:58,948 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 67ff5b7806924fe7a5265882eeb025eb, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733437128836 2024-12-05T22:18:58,958 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0088fb5c8612929b4093695e13cbc0fc#info#compaction#64 average throughput is 82.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:18:58,959 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/c56488accce7450f877bbbae6cb30f28 is 1080, key is row0062/info:/1733437068286/Put/seqid=0 2024-12-05T22:18:58,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741869_1045 (size=179609) 2024-12-05T22:18:58,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741869_1045 (size=179609) 2024-12-05T22:18:58,973 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/c56488accce7450f877bbbae6cb30f28 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/c56488accce7450f877bbbae6cb30f28 2024-12-05T22:18:58,978 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0088fb5c8612929b4093695e13cbc0fc/info of 0088fb5c8612929b4093695e13cbc0fc into c56488accce7450f877bbbae6cb30f28(size=175.4 K), total size for store is 175.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:18:58,978 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:18:58,978 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc., storeName=0088fb5c8612929b4093695e13cbc0fc/info, priority=13, startTime=1733437138946; duration=0sec 2024-12-05T22:18:58,978 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:18:58,978 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0088fb5c8612929b4093695e13cbc0fc:info 2024-12-05T22:18:59,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:19:00,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:19:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:19:00,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-05T22:19:00,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/000381a329f64dec9bc81e28dc6ef0ef is 1080, key is row0223/info:/1733437138924/Put/seqid=0 2024-12-05T22:19:00,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741870_1046 (size=12523) 2024-12-05T22:19:00,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741870_1046 (size=12523) 2024-12-05T22:19:00,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/000381a329f64dec9bc81e28dc6ef0ef 2024-12-05T22:19:00,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/000381a329f64dec9bc81e28dc6ef0ef as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/000381a329f64dec9bc81e28dc6ef0ef 2024-12-05T22:19:00,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/000381a329f64dec9bc81e28dc6ef0ef, entries=7, sequenceid=292, filesize=12.2 K 2024-12-05T22:19:00,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 0088fb5c8612929b4093695e13cbc0fc in 29ms, sequenceid=292, compaction requested=false 2024-12-05T22:19:00,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:19:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38777 {}] regionserver.HRegion(8581): Flush requested on 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:19:00,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-12-05T22:19:00,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/d304387c526e4bd3808be5f8c70551b1 is 1080, key is row0230/info:/1733437140932/Put/seqid=0 2024-12-05T22:19:00,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741871_1047 (size=21171) 2024-12-05T22:19:00,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741871_1047 (size=21171) 2024-12-05T22:19:00,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/d304387c526e4bd3808be5f8c70551b1 2024-12-05T22:19:00,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/d304387c526e4bd3808be5f8c70551b1 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d304387c526e4bd3808be5f8c70551b1 2024-12-05T22:19:00,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d304387c526e4bd3808be5f8c70551b1, entries=15, sequenceid=310, filesize=20.7 K 2024-12-05T22:19:00,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=12.61 KB/12912 for 0088fb5c8612929b4093695e13cbc0fc in 24ms, sequenceid=310, compaction requested=true 2024-12-05T22:19:00,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:19:00,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0088fb5c8612929b4093695e13cbc0fc:info, priority=-2147483648, current under compaction store size is 1 2024-12-05T22:19:00,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:19:00,985 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T22:19:00,986 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 213303 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T22:19:00,986 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1540): 0088fb5c8612929b4093695e13cbc0fc/info is initiating minor compaction (all files) 2024-12-05T22:19:00,987 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0088fb5c8612929b4093695e13cbc0fc/info in TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:19:00,987 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/c56488accce7450f877bbbae6cb30f28, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/000381a329f64dec9bc81e28dc6ef0ef, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d304387c526e4bd3808be5f8c70551b1] into tmpdir=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp, totalSize=208.3 K 2024-12-05T22:19:00,987 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting c56488accce7450f877bbbae6cb30f28, keycount=161, bloomtype=ROW, size=175.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733437068286 2024-12-05T22:19:00,988 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting 000381a329f64dec9bc81e28dc6ef0ef, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733437138924 2024-12-05T22:19:00,988 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] compactions.Compactor(224): Compacting d304387c526e4bd3808be5f8c70551b1, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733437140932 2024-12-05T22:19:01,003 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0088fb5c8612929b4093695e13cbc0fc#info#compaction#67 average throughput is 62.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T22:19:01,003 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/10a1b51107164289870f4d305674db3b is 1080, key is row0062/info:/1733437068286/Put/seqid=0 2024-12-05T22:19:01,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741872_1048 (size=203457) 2024-12-05T22:19:01,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741872_1048 (size=203457) 2024-12-05T22:19:01,016 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/10a1b51107164289870f4d305674db3b as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/10a1b51107164289870f4d305674db3b 2024-12-05T22:19:01,021 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0088fb5c8612929b4093695e13cbc0fc/info of 0088fb5c8612929b4093695e13cbc0fc into 10a1b51107164289870f4d305674db3b(size=198.7 K), total size for store is 198.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T22:19:01,021 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:19:01,022 INFO [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc., storeName=0088fb5c8612929b4093695e13cbc0fc/info, priority=13, startTime=1733437140985; duration=0sec 2024-12-05T22:19:01,022 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T22:19:01,022 DEBUG [RS:0;d029b80fd32e:38777-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0088fb5c8612929b4093695e13cbc0fc:info 2024-12-05T22:19:01,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:19:02,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:19:02,976 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-12-05T22:19:02,976 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38777%2C1733437034635.1733437142976 2024-12-05T22:19:02,984 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.1733437035062 with entries=309, filesize=306.80 KB; new WAL /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.1733437142976 2024-12-05T22:19:02,984 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44117:44117),(127.0.0.1/127.0.0.1:45521:45521)] 2024-12-05T22:19:02,984 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.1733437035062 is not closed yet, will try archiving it next time 2024-12-05T22:19:02,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741833_1009 (size=314174) 2024-12-05T22:19:02,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741833_1009 (size=314174) 2024-12-05T22:19:02,988 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 0088fb5c8612929b4093695e13cbc0fc 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-05T22:19:02,993 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/c73e0d9932824dd29ecc57c01cb8ecd6 is 1080, key is row0245/info:/1733437140962/Put/seqid=0 2024-12-05T22:19:02,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741874_1050 (size=17918) 2024-12-05T22:19:02,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741874_1050 (size=17918) 2024-12-05T22:19:03,000 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/c73e0d9932824dd29ecc57c01cb8ecd6 2024-12-05T22:19:03,005 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/.tmp/info/c73e0d9932824dd29ecc57c01cb8ecd6 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/c73e0d9932824dd29ecc57c01cb8ecd6 2024-12-05T22:19:03,009 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/c73e0d9932824dd29ecc57c01cb8ecd6, entries=12, sequenceid=326, filesize=17.5 K 2024-12-05T22:19:03,010 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for 0088fb5c8612929b4093695e13cbc0fc in 22ms, sequenceid=326, compaction requested=false 2024-12-05T22:19:03,010 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:19:03,010 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.21 KB heapSize=4.13 KB 2024-12-05T22:19:03,015 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/.tmp/info/0c957169c6ce47628bbf7ff0c248afb6 is 193, key is TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc./info:regioninfo/1733437071647/Put/seqid=0 2024-12-05T22:19:03,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741875_1051 (size=7803) 2024-12-05T22:19:03,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741875_1051 (size=7803) 2024-12-05T22:19:03,020 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.21 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/.tmp/info/0c957169c6ce47628bbf7ff0c248afb6 2024-12-05T22:19:03,027 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/.tmp/info/0c957169c6ce47628bbf7ff0c248afb6 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/info/0c957169c6ce47628bbf7ff0c248afb6 2024-12-05T22:19:03,033 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/info/0c957169c6ce47628bbf7ff0c248afb6, entries=16, sequenceid=24, filesize=7.6 K 2024-12-05T22:19:03,034 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~2.21 KB/2260, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=24, compaction requested=false 2024-12-05T22:19:03,034 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-12-05T22:19:03,034 INFO [Time-limited test {}] regionserver.HRegion(2837): Flushing 883cc4622e8d8175d333f4fa3e2982b4 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-05T22:19:03,056 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/namespace/883cc4622e8d8175d333f4fa3e2982b4/.tmp/info/ba852963f80644e1b73fc7f97465ee95 is 45, key is default/info:d/1733437035972/Put/seqid=0 2024-12-05T22:19:03,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741876_1052 (size=5037) 2024-12-05T22:19:03,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741876_1052 (size=5037) 2024-12-05T22:19:03,081 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/namespace/883cc4622e8d8175d333f4fa3e2982b4/.tmp/info/ba852963f80644e1b73fc7f97465ee95 2024-12-05T22:19:03,089 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/namespace/883cc4622e8d8175d333f4fa3e2982b4/.tmp/info/ba852963f80644e1b73fc7f97465ee95 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/namespace/883cc4622e8d8175d333f4fa3e2982b4/info/ba852963f80644e1b73fc7f97465ee95 2024-12-05T22:19:03,094 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/namespace/883cc4622e8d8175d333f4fa3e2982b4/info/ba852963f80644e1b73fc7f97465ee95, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T22:19:03,095 INFO [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 883cc4622e8d8175d333f4fa3e2982b4 in 61ms, sequenceid=6, compaction requested=false 2024-12-05T22:19:03,095 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 883cc4622e8d8175d333f4fa3e2982b4: 2024-12-05T22:19:03,095 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for dde3fcb9bf2d19e8c2b5cf489cea5b73: 2024-12-05T22:19:03,096 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C38777%2C1733437034635.1733437143095 2024-12-05T22:19:03,106 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.1733437142976 with entries=4, filesize=1.22 KB; new WAL /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.1733437143095 2024-12-05T22:19:03,106 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44117:44117),(127.0.0.1/127.0.0.1:45521:45521)] 2024-12-05T22:19:03,106 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.1733437142976 is not closed yet, will try archiving it next time 2024-12-05T22:19:03,107 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.1733437035062 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/oldWALs/d029b80fd32e%2C38777%2C1733437034635.1733437035062 2024-12-05T22:19:03,107 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T22:19:03,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741873_1049 (size=1255) 2024-12-05T22:19:03,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741873_1049 (size=1255) 2024-12-05T22:19:03,109 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635/d029b80fd32e%2C38777%2C1733437034635.1733437142976 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/oldWALs/d029b80fd32e%2C38777%2C1733437034635.1733437142976 2024-12-05T22:19:03,208 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-05T22:19:03,208 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-12-05T22:19:03,208 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x429e73de to 127.0.0.1:60973 2024-12-05T22:19:03,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:03,208 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T22:19:03,208 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=627199573, stopped=false 2024-12-05T22:19:03,208 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=d029b80fd32e,41859,1733437034541 2024-12-05T22:19:03,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:19:03,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:03,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:19:03,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:03,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-05T22:19:03,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:03,211 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,38777,1733437034635' ***** 2024-12-05T22:19:03,211 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-05T22:19:03,211 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:19:03,211 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(3579): Received CLOSE for 0088fb5c8612929b4093695e13cbc0fc 2024-12-05T22:19:03,212 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(3579): Received CLOSE for 883cc4622e8d8175d333f4fa3e2982b4 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(3579): Received CLOSE for dde3fcb9bf2d19e8c2b5cf489cea5b73 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,38777,1733437034635 2024-12-05T22:19:03,212 DEBUG [RS:0;d029b80fd32e:38777 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:03,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T22:19:03,212 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0088fb5c8612929b4093695e13cbc0fc, disabling compactions & flushes 2024-12-05T22:19:03,212 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-05T22:19:03,212 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:19:03,212 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:19:03,212 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. after waiting 0 ms 2024-12-05T22:19:03,212 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:19:03,213 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close 2024-12-05T22:19:03,213 DEBUG [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1603): Online Regions={0088fb5c8612929b4093695e13cbc0fc=TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc., 1588230740=hbase:meta,,1.1588230740, 883cc4622e8d8175d333f4fa3e2982b4=hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4., dde3fcb9bf2d19e8c2b5cf489cea5b73=TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73.} 2024-12-05T22:19:03,213 DEBUG [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1629): Waiting on 0088fb5c8612929b4093695e13cbc0fc, 1588230740, 883cc4622e8d8175d333f4fa3e2982b4, dde3fcb9bf2d19e8c2b5cf489cea5b73 2024-12-05T22:19:03,213 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:19:03,213 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:19:03,213 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:19:03,213 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:19:03,213 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:19:03,213 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7->hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/d772a2e4a82a421b905f019ba7af98f2-top, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d2e1ab2700984a3c87f94431621426de, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/TestLogRolling-testLogRolling=6166a9886eff2aa6d15d170434051ce7-4b8289c11b8f48a19ec7eb9df71d0ae8, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/98dba5760692407ea45e7c66425b5253, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/b8066221ed5843bcbadd71c85fc115c4, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/ea540428fa1a490d8588d87653420dad, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/0183a54d69594f3cadb95fb31c4d7cfb, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/035de0c4e74c4fb994b1c02725ff56c0, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/8b158fb989d447e7a9fe7ee6dddfc490, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/721a9b251e1840a889ef14b415a491be, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/90e792416c164c0198e49737f6f81a8f, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/56c1fb0178b64bf7b4029fe4ceb5bc46, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/3496028516274db6ba57ee71bd3b2c73, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/68c42d4a5c55442d8f45538d7779dded, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d0b2054b5eeb4b90936588de90eb3a58, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/936b30b8b6164685b36998231cbe6f55, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/c56488accce7450f877bbbae6cb30f28, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/67ff5b7806924fe7a5265882eeb025eb, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/000381a329f64dec9bc81e28dc6ef0ef, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d304387c526e4bd3808be5f8c70551b1] to archive 2024-12-05T22:19:03,215 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T22:19:03,217 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:19:03,220 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d2e1ab2700984a3c87f94431621426de to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d2e1ab2700984a3c87f94431621426de 2024-12-05T22:19:03,222 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/TestLogRolling-testLogRolling=6166a9886eff2aa6d15d170434051ce7-4b8289c11b8f48a19ec7eb9df71d0ae8 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/TestLogRolling-testLogRolling=6166a9886eff2aa6d15d170434051ce7-4b8289c11b8f48a19ec7eb9df71d0ae8 2024-12-05T22:19:03,224 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/98dba5760692407ea45e7c66425b5253 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/98dba5760692407ea45e7c66425b5253 2024-12-05T22:19:03,225 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/b8066221ed5843bcbadd71c85fc115c4 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/b8066221ed5843bcbadd71c85fc115c4 2024-12-05T22:19:03,227 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/ea540428fa1a490d8588d87653420dad to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/ea540428fa1a490d8588d87653420dad 2024-12-05T22:19:03,228 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/meta/1588230740/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2024-12-05T22:19:03,229 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/0183a54d69594f3cadb95fb31c4d7cfb to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/0183a54d69594f3cadb95fb31c4d7cfb 2024-12-05T22:19:03,229 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T22:19:03,229 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:19:03,229 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:19:03,230 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T22:19:03,231 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/035de0c4e74c4fb994b1c02725ff56c0 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/035de0c4e74c4fb994b1c02725ff56c0 2024-12-05T22:19:03,232 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/8b158fb989d447e7a9fe7ee6dddfc490 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/8b158fb989d447e7a9fe7ee6dddfc490 2024-12-05T22:19:03,234 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/721a9b251e1840a889ef14b415a491be to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/721a9b251e1840a889ef14b415a491be 2024-12-05T22:19:03,235 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/90e792416c164c0198e49737f6f81a8f to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/90e792416c164c0198e49737f6f81a8f 2024-12-05T22:19:03,237 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/56c1fb0178b64bf7b4029fe4ceb5bc46 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/56c1fb0178b64bf7b4029fe4ceb5bc46 2024-12-05T22:19:03,238 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/3496028516274db6ba57ee71bd3b2c73 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/3496028516274db6ba57ee71bd3b2c73 2024-12-05T22:19:03,240 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/68c42d4a5c55442d8f45538d7779dded to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/68c42d4a5c55442d8f45538d7779dded 2024-12-05T22:19:03,241 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d0b2054b5eeb4b90936588de90eb3a58 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d0b2054b5eeb4b90936588de90eb3a58 2024-12-05T22:19:03,243 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/936b30b8b6164685b36998231cbe6f55 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/936b30b8b6164685b36998231cbe6f55 2024-12-05T22:19:03,246 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/c56488accce7450f877bbbae6cb30f28 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/c56488accce7450f877bbbae6cb30f28 2024-12-05T22:19:03,249 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/67ff5b7806924fe7a5265882eeb025eb to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/67ff5b7806924fe7a5265882eeb025eb 2024-12-05T22:19:03,251 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/000381a329f64dec9bc81e28dc6ef0ef to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/000381a329f64dec9bc81e28dc6ef0ef 2024-12-05T22:19:03,252 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d304387c526e4bd3808be5f8c70551b1 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/info/d304387c526e4bd3808be5f8c70551b1 2024-12-05T22:19:03,257 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/0088fb5c8612929b4093695e13cbc0fc/recovered.edits/329.seqid, newMaxSeqId=329, maxSeqId=113 2024-12-05T22:19:03,258 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:19:03,258 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0088fb5c8612929b4093695e13cbc0fc: 2024-12-05T22:19:03,258 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733437070367.0088fb5c8612929b4093695e13cbc0fc. 2024-12-05T22:19:03,258 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 883cc4622e8d8175d333f4fa3e2982b4, disabling compactions & flushes 2024-12-05T22:19:03,258 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:19:03,258 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:19:03,258 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. after waiting 0 ms 2024-12-05T22:19:03,258 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:19:03,262 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/hbase/namespace/883cc4622e8d8175d333f4fa3e2982b4/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T22:19:03,265 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:19:03,265 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 883cc4622e8d8175d333f4fa3e2982b4: 2024-12-05T22:19:03,265 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733437035551.883cc4622e8d8175d333f4fa3e2982b4. 2024-12-05T22:19:03,265 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing dde3fcb9bf2d19e8c2b5cf489cea5b73, disabling compactions & flushes 2024-12-05T22:19:03,265 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:19:03,265 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:19:03,265 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. after waiting 0 ms 2024-12-05T22:19:03,265 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:19:03,265 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7->hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/6166a9886eff2aa6d15d170434051ce7/info/d772a2e4a82a421b905f019ba7af98f2-bottom] to archive 2024-12-05T22:19:03,266 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T22:19:03,268 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7 to hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/archive/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/info/d772a2e4a82a421b905f019ba7af98f2.6166a9886eff2aa6d15d170434051ce7 2024-12-05T22:19:03,271 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/data/default/TestLogRolling-testLogRolling/dde3fcb9bf2d19e8c2b5cf489cea5b73/recovered.edits/118.seqid, newMaxSeqId=118, maxSeqId=113 2024-12-05T22:19:03,272 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:19:03,272 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for dde3fcb9bf2d19e8c2b5cf489cea5b73: 2024-12-05T22:19:03,272 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733437070367.dde3fcb9bf2d19e8c2b5cf489cea5b73. 2024-12-05T22:19:03,413 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,38777,1733437034635; all regions closed. 2024-12-05T22:19:03,413 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635 2024-12-05T22:19:03,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741834_1010 (size=9351) 2024-12-05T22:19:03,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741834_1010 (size=9351) 2024-12-05T22:19:03,417 DEBUG [RS:0;d029b80fd32e:38777 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/oldWALs 2024-12-05T22:19:03,417 INFO [RS:0;d029b80fd32e:38777 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog d029b80fd32e%2C38777%2C1733437034635.meta:.meta(num 1733437035492) 2024-12-05T22:19:03,418 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/WALs/d029b80fd32e,38777,1733437034635 2024-12-05T22:19:03,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741877_1053 (size=1071) 2024-12-05T22:19:03,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741877_1053 (size=1071) 2024-12-05T22:19:03,421 DEBUG [RS:0;d029b80fd32e:38777 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/oldWALs 2024-12-05T22:19:03,421 INFO [RS:0;d029b80fd32e:38777 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog d029b80fd32e%2C38777%2C1733437034635:(num 1733437143095) 2024-12-05T22:19:03,421 DEBUG [RS:0;d029b80fd32e:38777 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:03,422 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:19:03,422 INFO [RS:0;d029b80fd32e:38777 {}] hbase.ChoreService(370): Chore service for: regionserver/d029b80fd32e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-05T22:19:03,422 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:19:03,422 INFO [RS:0;d029b80fd32e:38777 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38777 2024-12-05T22:19:03,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d029b80fd32e,38777,1733437034635 2024-12-05T22:19:03,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:19:03,425 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d029b80fd32e,38777,1733437034635] 2024-12-05T22:19:03,425 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing d029b80fd32e,38777,1733437034635; numProcessing=1 2024-12-05T22:19:03,427 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/d029b80fd32e,38777,1733437034635 already deleted, retry=false 2024-12-05T22:19:03,427 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; d029b80fd32e,38777,1733437034635 expired; onlineServers=0 2024-12-05T22:19:03,427 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,41859,1733437034541' ***** 2024-12-05T22:19:03,427 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T22:19:03,427 DEBUG [M:0;d029b80fd32e:41859 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200f5468, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:19:03,427 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,41859,1733437034541 2024-12-05T22:19:03,427 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,41859,1733437034541; all regions closed. 2024-12-05T22:19:03,427 DEBUG [M:0;d029b80fd32e:41859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:03,427 DEBUG [M:0;d029b80fd32e:41859 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T22:19:03,427 DEBUG [M:0;d029b80fd32e:41859 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T22:19:03,427 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T22:19:03,427 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733437034874 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733437034874,5,FailOnTimeoutGroup] 2024-12-05T22:19:03,427 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733437034874 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733437034874,5,FailOnTimeoutGroup] 2024-12-05T22:19:03,427 INFO [M:0;d029b80fd32e:41859 {}] hbase.ChoreService(370): Chore service for: master/d029b80fd32e:0 had [] on shutdown 2024-12-05T22:19:03,428 DEBUG [M:0;d029b80fd32e:41859 {}] master.HMaster(1733): Stopping service threads 2024-12-05T22:19:03,428 INFO [M:0;d029b80fd32e:41859 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T22:19:03,428 INFO [M:0;d029b80fd32e:41859 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T22:19:03,428 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T22:19:03,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T22:19:03,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:03,430 DEBUG [M:0;d029b80fd32e:41859 {}] zookeeper.ZKUtil(347): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T22:19:03,430 WARN [M:0;d029b80fd32e:41859 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T22:19:03,430 INFO [M:0;d029b80fd32e:41859 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-05T22:19:03,430 INFO [M:0;d029b80fd32e:41859 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T22:19:03,430 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:19:03,430 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:03,430 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:03,430 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:19:03,430 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:19:03,430 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:03,430 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=66.44 KB heapSize=81.69 KB 2024-12-05T22:19:03,445 DEBUG [M:0;d029b80fd32e:41859 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74080ccd11e74acebaa8fc102e4293eb is 82, key is hbase:meta,,1/info:regioninfo/1733437035514/Put/seqid=0 2024-12-05T22:19:03,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741878_1054 (size=5672) 2024-12-05T22:19:03,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741878_1054 (size=5672) 2024-12-05T22:19:03,451 INFO [M:0;d029b80fd32e:41859 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74080ccd11e74acebaa8fc102e4293eb 2024-12-05T22:19:03,476 DEBUG [M:0;d029b80fd32e:41859 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b3317c487f774922aa70320efc1525b2 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733437036921/Put/seqid=0 2024-12-05T22:19:03,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741879_1055 (size=7277) 2024-12-05T22:19:03,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741879_1055 (size=7277) 2024-12-05T22:19:03,488 INFO [M:0;d029b80fd32e:41859 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65.83 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b3317c487f774922aa70320efc1525b2 2024-12-05T22:19:03,492 INFO [M:0;d029b80fd32e:41859 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b3317c487f774922aa70320efc1525b2 2024-12-05T22:19:03,513 DEBUG [M:0;d029b80fd32e:41859 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/38c214896fd04a079b4cca8e3395a04e is 69, key is d029b80fd32e,38777,1733437034635/rs:state/1733437034911/Put/seqid=0 2024-12-05T22:19:03,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741880_1056 (size=5156) 2024-12-05T22:19:03,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741880_1056 (size=5156) 2024-12-05T22:19:03,520 INFO [M:0;d029b80fd32e:41859 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/38c214896fd04a079b4cca8e3395a04e 2024-12-05T22:19:03,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:19:03,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38777-0x10184b3fa890001, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:19:03,526 INFO [RS:0;d029b80fd32e:38777 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,38777,1733437034635; zookeeper connection closed. 2024-12-05T22:19:03,527 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@33dafe45 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@33dafe45 2024-12-05T22:19:03,527 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T22:19:03,542 DEBUG [M:0;d029b80fd32e:41859 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0b5143104f634d398fd5577fdf03c765 is 52, key is load_balancer_on/state:d/1733437036114/Put/seqid=0 2024-12-05T22:19:03,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741881_1057 (size=5056) 2024-12-05T22:19:03,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741881_1057 (size=5056) 2024-12-05T22:19:03,547 INFO [M:0;d029b80fd32e:41859 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0b5143104f634d398fd5577fdf03c765 2024-12-05T22:19:03,552 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74080ccd11e74acebaa8fc102e4293eb as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74080ccd11e74acebaa8fc102e4293eb 2024-12-05T22:19:03,556 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74080ccd11e74acebaa8fc102e4293eb, entries=8, sequenceid=164, filesize=5.5 K 2024-12-05T22:19:03,557 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b3317c487f774922aa70320efc1525b2 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b3317c487f774922aa70320efc1525b2 2024-12-05T22:19:03,561 INFO [M:0;d029b80fd32e:41859 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b3317c487f774922aa70320efc1525b2 2024-12-05T22:19:03,561 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b3317c487f774922aa70320efc1525b2, entries=18, sequenceid=164, filesize=7.1 K 2024-12-05T22:19:03,561 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/38c214896fd04a079b4cca8e3395a04e as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/38c214896fd04a079b4cca8e3395a04e 2024-12-05T22:19:03,565 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/38c214896fd04a079b4cca8e3395a04e, entries=1, sequenceid=164, filesize=5.0 K 2024-12-05T22:19:03,566 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0b5143104f634d398fd5577fdf03c765 as hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0b5143104f634d398fd5577fdf03c765 2024-12-05T22:19:03,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:19:03,571 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46345/user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0b5143104f634d398fd5577fdf03c765, entries=1, sequenceid=164, filesize=4.9 K 2024-12-05T22:19:03,573 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HRegion(3040): Finished flush of dataSize ~66.44 KB/68031, heapSize ~81.63 KB/83584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 143ms, sequenceid=164, compaction requested=false 2024-12-05T22:19:03,574 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:03,574 DEBUG [M:0;d029b80fd32e:41859 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:19:03,575 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/35a41d5d-8a70-2c9f-9f4a-aec9399870dc/MasterData/WALs/d029b80fd32e,41859,1733437034541 2024-12-05T22:19:03,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46287 is added to blk_1073741830_1006 (size=79260) 2024-12-05T22:19:03,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36311 is added to blk_1073741830_1006 (size=79260) 2024-12-05T22:19:03,577 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:19:03,577 INFO [M:0;d029b80fd32e:41859 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-05T22:19:03,577 INFO [M:0;d029b80fd32e:41859 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:41859 2024-12-05T22:19:03,579 DEBUG [M:0;d029b80fd32e:41859 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/d029b80fd32e,41859,1733437034541 already deleted, retry=false 2024-12-05T22:19:03,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:19:03,681 INFO [M:0;d029b80fd32e:41859 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,41859,1733437034541; zookeeper connection closed. 2024-12-05T22:19:03,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41859-0x10184b3fa890000, quorum=127.0.0.1:60973, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:19:03,684 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1e6d7f99{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:19:03,685 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@710c1196{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:19:03,685 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:19:03,685 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4105b643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:19:03,685 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@548d4f02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.log.dir/,STOPPED} 2024-12-05T22:19:03,687 WARN [BP-495755302-172.17.0.2-1733437033442 heartbeating to localhost/127.0.0.1:46345 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:19:03,687 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:19:03,687 WARN [BP-495755302-172.17.0.2-1733437033442 heartbeating to localhost/127.0.0.1:46345 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-495755302-172.17.0.2-1733437033442 (Datanode Uuid 4e8ceff1-9976-4027-bd50-ad575a0acff6) service to localhost/127.0.0.1:46345 2024-12-05T22:19:03,687 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:19:03,688 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/dfs/data/data3/current/BP-495755302-172.17.0.2-1733437033442 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:19:03,688 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/dfs/data/data4/current/BP-495755302-172.17.0.2-1733437033442 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:19:03,688 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:19:03,690 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4be9feb9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:19:03,691 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@73d21e1d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:19:03,691 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:19:03,691 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29238c79{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:19:03,691 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@65cd88a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.log.dir/,STOPPED} 2024-12-05T22:19:03,693 WARN [BP-495755302-172.17.0.2-1733437033442 heartbeating to localhost/127.0.0.1:46345 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:19:03,693 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:19:03,693 WARN [BP-495755302-172.17.0.2-1733437033442 heartbeating to localhost/127.0.0.1:46345 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-495755302-172.17.0.2-1733437033442 (Datanode Uuid ef54fe69-bf95-4af9-81cb-e98911855bf1) service to localhost/127.0.0.1:46345 2024-12-05T22:19:03,693 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:19:03,693 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/dfs/data/data1/current/BP-495755302-172.17.0.2-1733437033442 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:19:03,693 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/cluster_7b077835-5875-b5d9-6e38-8142a4f43d91/dfs/data/data2/current/BP-495755302-172.17.0.2-1733437033442 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:19:03,694 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:19:03,701 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1cd1d5c9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:19:03,702 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ff97e6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:19:03,702 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:19:03,702 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3fd2286c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:19:03,702 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52e6c5a0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.log.dir/,STOPPED} 2024-12-05T22:19:03,710 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-05T22:19:03,736 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-05T22:19:03,745 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=125 (was 111) - Thread LEAK? -, OpenFileDescriptor=493 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=238 (was 348), ProcessCount=11 (was 11), AvailableMemoryMB=3586 (was 2810) - AvailableMemoryMB LEAK? - 2024-12-05T22:19:03,753 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=126, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=238, ProcessCount=11, AvailableMemoryMB=3586 2024-12-05T22:19:03,753 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T22:19:03,753 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.log.dir so I do NOT create it in target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/ff06b78e-ed6a-3964-7947-d3e7af7d19e1/hadoop.tmp.dir so I do NOT create it in target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891, deleteOnExit=true 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/test.cache.data in system properties and HBase conf 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/hadoop.log.dir in system properties and HBase conf 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-05T22:19:03,754 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:19:03,754 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T22:19:03,755 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T22:19:03,755 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T22:19:03,755 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:19:03,755 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T22:19:03,755 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/nfs.dump.dir in system properties and HBase conf 2024-12-05T22:19:03,755 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/java.io.tmpdir in system properties and HBase conf 2024-12-05T22:19:03,755 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T22:19:03,755 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T22:19:03,755 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T22:19:03,768 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:19:03,883 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:19:03,906 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:19:03,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:19:03,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:19:03,920 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:19:03,920 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:19:03,921 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6db6e54e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:19:03,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@693f2ee7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:19:04,066 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a3e1974{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/java.io.tmpdir/jetty-localhost-38915-hadoop-hdfs-3_4_1-tests_jar-_-any-12000099822196523760/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:19:04,067 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@63867263{HTTP/1.1, (http/1.1)}{localhost:38915} 2024-12-05T22:19:04,067 INFO [Time-limited test {}] server.Server(415): Started @397785ms 2024-12-05T22:19:04,080 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-05T22:19:04,159 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:19:04,163 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:19:04,164 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:19:04,164 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:19:04,164 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T22:19:04,165 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d6c116b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:19:04,165 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2990422d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:19:04,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@37530d4a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/java.io.tmpdir/jetty-localhost-34341-hadoop-hdfs-3_4_1-tests_jar-_-any-10275913175077391119/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:19:04,286 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5daa6290{HTTP/1.1, (http/1.1)}{localhost:34341} 2024-12-05T22:19:04,286 INFO [Time-limited test {}] server.Server(415): Started @398004ms 2024-12-05T22:19:04,288 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:19:04,335 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T22:19:04,340 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T22:19:04,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T22:19:04,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T22:19:04,341 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T22:19:04,342 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69321fa9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/hadoop.log.dir/,AVAILABLE} 2024-12-05T22:19:04,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33d3a777{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T22:19:04,410 WARN [Thread-2207 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/dfs/data/data1/current/BP-1006552126-172.17.0.2-1733437143795/current, will proceed with Du for space computation calculation, 2024-12-05T22:19:04,410 WARN [Thread-2208 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/dfs/data/data2/current/BP-1006552126-172.17.0.2-1733437143795/current, will proceed with Du for space computation calculation, 2024-12-05T22:19:04,435 WARN [Thread-2186 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:19:04,438 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe18d10ee421ab2d0 with lease ID 0x547cb905958f6012: Processing first storage report for DS-f2d0183e-df63-4115-be05-a17e2fdcb651 from datanode DatanodeRegistration(127.0.0.1:41825, datanodeUuid=6c92c92f-7698-4d2e-b6c2-af6797079ce7, infoPort=36453, infoSecurePort=0, ipcPort=34335, storageInfo=lv=-57;cid=testClusterID;nsid=490189907;c=1733437143795) 2024-12-05T22:19:04,439 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe18d10ee421ab2d0 with lease ID 0x547cb905958f6012: from storage DS-f2d0183e-df63-4115-be05-a17e2fdcb651 node DatanodeRegistration(127.0.0.1:41825, datanodeUuid=6c92c92f-7698-4d2e-b6c2-af6797079ce7, infoPort=36453, infoSecurePort=0, ipcPort=34335, storageInfo=lv=-57;cid=testClusterID;nsid=490189907;c=1733437143795), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T22:19:04,439 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe18d10ee421ab2d0 with lease ID 0x547cb905958f6012: Processing first storage report for DS-d8cf2e2f-74e4-4b4b-815e-85cb5d59ae4b from datanode DatanodeRegistration(127.0.0.1:41825, datanodeUuid=6c92c92f-7698-4d2e-b6c2-af6797079ce7, infoPort=36453, infoSecurePort=0, ipcPort=34335, storageInfo=lv=-57;cid=testClusterID;nsid=490189907;c=1733437143795) 2024-12-05T22:19:04,439 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe18d10ee421ab2d0 with lease ID 0x547cb905958f6012: from storage DS-d8cf2e2f-74e4-4b4b-815e-85cb5d59ae4b node DatanodeRegistration(127.0.0.1:41825, datanodeUuid=6c92c92f-7698-4d2e-b6c2-af6797079ce7, infoPort=36453, infoSecurePort=0, ipcPort=34335, storageInfo=lv=-57;cid=testClusterID;nsid=490189907;c=1733437143795), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:19:04,493 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420757c0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/java.io.tmpdir/jetty-localhost-41553-hadoop-hdfs-3_4_1-tests_jar-_-any-11885433015206110679/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:19:04,494 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7ef49dde{HTTP/1.1, (http/1.1)}{localhost:41553} 2024-12-05T22:19:04,494 INFO [Time-limited test {}] server.Server(415): Started @398212ms 2024-12-05T22:19:04,495 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T22:19:04,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:19:04,596 WARN [Thread-2233 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/dfs/data/data3/current/BP-1006552126-172.17.0.2-1733437143795/current, will proceed with Du for space computation calculation, 2024-12-05T22:19:04,596 WARN [Thread-2234 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/dfs/data/data4/current/BP-1006552126-172.17.0.2-1733437143795/current, will proceed with Du for space computation calculation, 2024-12-05T22:19:04,614 WARN [Thread-2222 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T22:19:04,616 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x406e99674c86fde9 with lease ID 0x547cb905958f6013: Processing first storage report for DS-b363f88c-ecf4-4e74-8caf-0c1b025358a5 from datanode DatanodeRegistration(127.0.0.1:44207, datanodeUuid=e43499f8-3343-46ad-949c-181fe36cc1a2, infoPort=45941, infoSecurePort=0, ipcPort=33043, storageInfo=lv=-57;cid=testClusterID;nsid=490189907;c=1733437143795) 2024-12-05T22:19:04,616 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x406e99674c86fde9 with lease ID 0x547cb905958f6013: from storage DS-b363f88c-ecf4-4e74-8caf-0c1b025358a5 node DatanodeRegistration(127.0.0.1:44207, datanodeUuid=e43499f8-3343-46ad-949c-181fe36cc1a2, infoPort=45941, infoSecurePort=0, ipcPort=33043, storageInfo=lv=-57;cid=testClusterID;nsid=490189907;c=1733437143795), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:19:04,616 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x406e99674c86fde9 with lease ID 0x547cb905958f6013: Processing first storage report for DS-5c7d2502-c527-4fe8-848e-80b011184c7b from datanode DatanodeRegistration(127.0.0.1:44207, datanodeUuid=e43499f8-3343-46ad-949c-181fe36cc1a2, infoPort=45941, infoSecurePort=0, ipcPort=33043, storageInfo=lv=-57;cid=testClusterID;nsid=490189907;c=1733437143795) 2024-12-05T22:19:04,616 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x406e99674c86fde9 with lease ID 0x547cb905958f6013: from storage DS-5c7d2502-c527-4fe8-848e-80b011184c7b node DatanodeRegistration(127.0.0.1:44207, datanodeUuid=e43499f8-3343-46ad-949c-181fe36cc1a2, infoPort=45941, infoSecurePort=0, ipcPort=33043, storageInfo=lv=-57;cid=testClusterID;nsid=490189907;c=1733437143795), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T22:19:04,620 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7 2024-12-05T22:19:04,624 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/zookeeper_0, clientPort=53795, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T22:19:04,625 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=53795 2024-12-05T22:19:04,625 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:19:04,627 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:19:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:19:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741825_1001 (size=7) 2024-12-05T22:19:04,637 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394 with version=8 2024-12-05T22:19:04,637 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:37467/user/jenkins/test-data/322734b4-d16f-7a3d-cfb3-0ad60d408866/hbase-staging 2024-12-05T22:19:04,639 INFO [Time-limited test {}] client.ConnectionUtils(129): master/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:19:04,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:19:04,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:19:04,640 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:19:04,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:19:04,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:19:04,640 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:19:04,640 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:19:04,641 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45153 2024-12-05T22:19:04,641 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:19:04,643 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:19:04,646 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:45153 connecting to ZooKeeper ensemble=127.0.0.1:53795 2024-12-05T22:19:04,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:451530x0, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:19:04,654 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45153-0x10184b5a8be0000 connected 2024-12-05T22:19:04,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:19:04,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:19:04,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:19:04,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45153 2024-12-05T22:19:04,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45153 2024-12-05T22:19:04,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45153 2024-12-05T22:19:04,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45153 2024-12-05T22:19:04,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45153 2024-12-05T22:19:04,674 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394, hbase.cluster.distributed=false 2024-12-05T22:19:04,693 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/d029b80fd32e:0 server-side Connection retries=45 2024-12-05T22:19:04,694 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:19:04,694 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T22:19:04,694 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T22:19:04,694 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T22:19:04,694 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T22:19:04,694 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T22:19:04,694 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T22:19:04,695 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:33085 2024-12-05T22:19:04,695 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T22:19:04,696 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T22:19:04,696 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:19:04,699 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:19:04,702 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:33085 connecting to ZooKeeper ensemble=127.0.0.1:53795 2024-12-05T22:19:04,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330850x0, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T22:19:04,704 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33085-0x10184b5a8be0001 connected 2024-12-05T22:19:04,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:19:04,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:19:04,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T22:19:04,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33085 2024-12-05T22:19:04,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33085 2024-12-05T22:19:04,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33085 2024-12-05T22:19:04,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33085 2024-12-05T22:19:04,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33085 2024-12-05T22:19:04,708 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/d029b80fd32e,45153,1733437144639 2024-12-05T22:19:04,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:19:04,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:19:04,710 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d029b80fd32e,45153,1733437144639 2024-12-05T22:19:04,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:19:04,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T22:19:04,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,717 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:19:04,717 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d029b80fd32e,45153,1733437144639 from backup master directory 2024-12-05T22:19:04,722 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d029b80fd32e:45153 2024-12-05T22:19:04,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d029b80fd32e,45153,1733437144639 2024-12-05T22:19:04,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:19:04,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T22:19:04,722 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:19:04,722 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d029b80fd32e,45153,1733437144639 2024-12-05T22:19:04,722 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T22:19:04,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:19:04,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741826_1002 (size=42) 2024-12-05T22:19:04,742 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/hbase.id with ID: e6971aef-7ef8-4bc6-8f61-3d44ebfe6a93 2024-12-05T22:19:04,753 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:19:04,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:19:04,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741827_1003 (size=196) 2024-12-05T22:19:04,767 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T22:19:04,768 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T22:19:04,768 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:19:04,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:19:04,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741828_1004 (size=1189) 2024-12-05T22:19:04,781 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store 2024-12-05T22:19:04,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:19:04,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741829_1005 (size=34) 2024-12-05T22:19:04,792 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:19:04,792 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:19:04,792 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:04,792 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:04,792 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:19:04,792 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:04,792 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:04,792 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:19:04,794 WARN [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/.initializing 2024-12-05T22:19:04,794 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/WALs/d029b80fd32e,45153,1733437144639 2024-12-05T22:19:04,797 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C45153%2C1733437144639, suffix=, logDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/WALs/d029b80fd32e,45153,1733437144639, archiveDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/oldWALs, maxLogs=10 2024-12-05T22:19:04,798 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C45153%2C1733437144639.1733437144797 2024-12-05T22:19:04,805 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/WALs/d029b80fd32e,45153,1733437144639/d029b80fd32e%2C45153%2C1733437144639.1733437144797 2024-12-05T22:19:04,805 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45941:45941),(127.0.0.1/127.0.0.1:36453:36453)] 2024-12-05T22:19:04,805 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:19:04,806 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:19:04,806 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:19:04,806 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:19:04,807 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:19:04,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T22:19:04,808 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:04,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:19:04,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:19:04,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T22:19:04,810 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:04,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:19:04,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:19:04,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T22:19:04,812 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:04,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:19:04,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:19:04,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T22:19:04,813 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:04,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:19:04,814 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:19:04,815 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:19:04,816 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T22:19:04,817 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T22:19:04,819 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:19:04,820 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=724523, jitterRate=-0.07872229814529419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T22:19:04,820 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:19:04,820 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T22:19:04,823 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ff10f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:19:04,824 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-05T22:19:04,824 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T22:19:04,824 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T22:19:04,824 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T22:19:04,824 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T22:19:04,825 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec 2024-12-05T22:19:04,825 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T22:19:04,828 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T22:19:04,829 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T22:19:04,830 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-05T22:19:04,830 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T22:19:04,831 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T22:19:04,832 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-05T22:19:04,832 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T22:19:04,833 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T22:19:04,835 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-05T22:19:04,836 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T22:19:04,837 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T22:19:04,838 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T22:19:04,839 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T22:19:04,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:19:04,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T22:19:04,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,841 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=d029b80fd32e,45153,1733437144639, sessionid=0x10184b5a8be0000, setting cluster-up flag (Was=false) 2024-12-05T22:19:04,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,851 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T22:19:04,852 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,45153,1733437144639 2024-12-05T22:19:04,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:04,860 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T22:19:04,861 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d029b80fd32e,45153,1733437144639 2024-12-05T22:19:04,864 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-05T22:19:04,864 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-05T22:19:04,864 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T22:19:04,864 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d029b80fd32e,45153,1733437144639 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T22:19:04,865 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:19:04,865 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:19:04,865 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:19:04,865 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d029b80fd32e:0, corePoolSize=5, maxPoolSize=5 2024-12-05T22:19:04,865 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d029b80fd32e:0, corePoolSize=10, maxPoolSize=10 2024-12-05T22:19:04,865 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,865 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:19:04,865 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,866 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733437174866 2024-12-05T22:19:04,866 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T22:19:04,866 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T22:19:04,866 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T22:19:04,867 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T22:19:04,867 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T22:19:04,867 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T22:19:04,867 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:19:04,867 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-05T22:19:04,867 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,867 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T22:19:04,867 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T22:19:04,867 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T22:19:04,868 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T22:19:04,868 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T22:19:04,868 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:04,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733437144868,5,FailOnTimeoutGroup] 2024-12-05T22:19:04,868 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733437144868,5,FailOnTimeoutGroup] 2024-12-05T22:19:04,868 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,868 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T22:19:04,868 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,868 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,868 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:19:04,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:19:04,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741831_1007 (size=1039) 2024-12-05T22:19:04,875 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-05T22:19:04,875 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394 2024-12-05T22:19:04,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:19:04,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741832_1008 (size=32) 2024-12-05T22:19:04,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:19:04,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:19:04,885 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:19:04,885 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:04,885 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:19:04,885 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:19:04,886 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:19:04,886 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:04,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:19:04,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:19:04,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:19:04,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:04,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:19:04,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740 2024-12-05T22:19:04,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740 2024-12-05T22:19:04,891 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:19:04,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:19:04,895 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:19:04,895 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=800656, jitterRate=0.018087655305862427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:19:04,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:19:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:19:04,896 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:19:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:19:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:19:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:19:04,896 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:19:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:19:04,897 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T22:19:04,897 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-05T22:19:04,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T22:19:04,898 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T22:19:04,899 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T22:19:04,931 INFO [regionserver/d029b80fd32e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:19:04,936 DEBUG [RS:0;d029b80fd32e:33085 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d029b80fd32e:33085 2024-12-05T22:19:04,937 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1008): ClusterId : e6971aef-7ef8-4bc6-8f61-3d44ebfe6a93 2024-12-05T22:19:04,937 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T22:19:04,939 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T22:19:04,939 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T22:19:04,941 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T22:19:04,941 DEBUG [RS:0;d029b80fd32e:33085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b8bf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:19:04,944 DEBUG [RS:0;d029b80fd32e:33085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268df436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:19:04,944 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-05T22:19:04,944 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-05T22:19:04,945 DEBUG [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-05T22:19:04,945 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(3073): reportForDuty to master=d029b80fd32e,45153,1733437144639 with isa=d029b80fd32e/172.17.0.2:33085, startcode=1733437144693 2024-12-05T22:19:04,945 DEBUG [RS:0;d029b80fd32e:33085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T22:19:04,957 INFO [RS-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58883, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T22:19:04,957 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45153 {}] master.ServerManager(332): Checking decommissioned status of RegionServer d029b80fd32e,33085,1733437144693 2024-12-05T22:19:04,957 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45153 {}] master.ServerManager(486): Registering regionserver=d029b80fd32e,33085,1733437144693 2024-12-05T22:19:04,959 DEBUG [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394 2024-12-05T22:19:04,959 DEBUG [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:40725 2024-12-05T22:19:04,959 DEBUG [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-05T22:19:04,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:19:04,966 DEBUG [RS:0;d029b80fd32e:33085 {}] zookeeper.ZKUtil(111): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d029b80fd32e,33085,1733437144693 2024-12-05T22:19:04,966 WARN [RS:0;d029b80fd32e:33085 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T22:19:04,966 INFO [RS:0;d029b80fd32e:33085 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:19:04,966 DEBUG [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/d029b80fd32e,33085,1733437144693 2024-12-05T22:19:04,969 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d029b80fd32e,33085,1733437144693] 2024-12-05T22:19:04,973 DEBUG [RS:0;d029b80fd32e:33085 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-05T22:19:04,973 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T22:19:04,976 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T22:19:04,977 INFO [RS:0;d029b80fd32e:33085 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T22:19:04,977 INFO [RS:0;d029b80fd32e:33085 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,977 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-05T22:19:04,978 INFO [RS:0;d029b80fd32e:33085 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d029b80fd32e:0, corePoolSize=2, maxPoolSize=2 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d029b80fd32e:0, corePoolSize=1, maxPoolSize=1 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:19:04,978 DEBUG [RS:0;d029b80fd32e:33085 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d029b80fd32e:0, corePoolSize=3, maxPoolSize=3 2024-12-05T22:19:04,980 INFO [RS:0;d029b80fd32e:33085 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,980 INFO [RS:0;d029b80fd32e:33085 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,980 INFO [RS:0;d029b80fd32e:33085 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,980 INFO [RS:0;d029b80fd32e:33085 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:04,980 INFO [RS:0;d029b80fd32e:33085 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,33085,1733437144693-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:19:04,998 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T22:19:04,998 INFO [RS:0;d029b80fd32e:33085 {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,33085,1733437144693-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:05,013 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.Replication(204): d029b80fd32e,33085,1733437144693 started 2024-12-05T22:19:05,013 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1767): Serving as d029b80fd32e,33085,1733437144693, RpcServer on d029b80fd32e/172.17.0.2:33085, sessionid=0x10184b5a8be0001 2024-12-05T22:19:05,013 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T22:19:05,013 DEBUG [RS:0;d029b80fd32e:33085 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d029b80fd32e,33085,1733437144693 2024-12-05T22:19:05,013 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,33085,1733437144693' 2024-12-05T22:19:05,013 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T22:19:05,013 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T22:19:05,014 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T22:19:05,014 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T22:19:05,014 DEBUG [RS:0;d029b80fd32e:33085 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d029b80fd32e,33085,1733437144693 2024-12-05T22:19:05,014 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd029b80fd32e,33085,1733437144693' 2024-12-05T22:19:05,014 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T22:19:05,014 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T22:19:05,014 DEBUG [RS:0;d029b80fd32e:33085 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T22:19:05,014 INFO [RS:0;d029b80fd32e:33085 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T22:19:05,014 INFO [RS:0;d029b80fd32e:33085 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T22:19:05,049 WARN [d029b80fd32e:45153 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-05T22:19:05,117 INFO [RS:0;d029b80fd32e:33085 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C33085%2C1733437144693, suffix=, logDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/d029b80fd32e,33085,1733437144693, archiveDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/oldWALs, maxLogs=32 2024-12-05T22:19:05,118 INFO [RS:0;d029b80fd32e:33085 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C33085%2C1733437144693.1733437145118 2024-12-05T22:19:05,130 INFO [RS:0;d029b80fd32e:33085 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/d029b80fd32e,33085,1733437144693/d029b80fd32e%2C33085%2C1733437144693.1733437145118 2024-12-05T22:19:05,130 DEBUG [RS:0;d029b80fd32e:33085 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36453:36453),(127.0.0.1/127.0.0.1:45941:45941)] 2024-12-05T22:19:05,299 DEBUG [d029b80fd32e:45153 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T22:19:05,299 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d029b80fd32e,33085,1733437144693 2024-12-05T22:19:05,300 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,33085,1733437144693, state=OPENING 2024-12-05T22:19:05,302 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T22:19:05,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:05,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:05,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:19:05,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:19:05,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=d029b80fd32e,33085,1733437144693}] 2024-12-05T22:19:05,456 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33085,1733437144693 2024-12-05T22:19:05,457 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T22:19:05,459 INFO [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37182, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T22:19:05,461 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-05T22:19:05,462 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:19:05,463 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d029b80fd32e%2C33085%2C1733437144693.meta, suffix=.meta, logDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/d029b80fd32e,33085,1733437144693, archiveDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/oldWALs, maxLogs=32 2024-12-05T22:19:05,463 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d029b80fd32e%2C33085%2C1733437144693.meta.1733437145463.meta 2024-12-05T22:19:05,472 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/d029b80fd32e,33085,1733437144693/d029b80fd32e%2C33085%2C1733437144693.meta.1733437145463.meta 2024-12-05T22:19:05,472 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36453:36453),(127.0.0.1/127.0.0.1:45941:45941)] 2024-12-05T22:19:05,472 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:19:05,473 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T22:19:05,473 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T22:19:05,473 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T22:19:05,473 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T22:19:05,473 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:19:05,473 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-05T22:19:05,473 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-05T22:19:05,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T22:19:05,475 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T22:19:05,475 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:05,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:19:05,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T22:19:05,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T22:19:05,476 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:05,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:19:05,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T22:19:05,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T22:19:05,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:05,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T22:19:05,478 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740 2024-12-05T22:19:05,478 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740 2024-12-05T22:19:05,479 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T22:19:05,480 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T22:19:05,481 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713102, jitterRate=-0.09324492514133453}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T22:19:05,481 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T22:19:05,482 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733437145456 2024-12-05T22:19:05,483 DEBUG [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T22:19:05,483 INFO [RS_OPEN_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-05T22:19:05,484 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,33085,1733437144693 2024-12-05T22:19:05,485 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d029b80fd32e,33085,1733437144693, state=OPEN 2024-12-05T22:19:05,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:19:05,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T22:19:05,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:19:05,492 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T22:19:05,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T22:19:05,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=d029b80fd32e,33085,1733437144693 in 188 msec 2024-12-05T22:19:05,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T22:19:05,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 597 msec 2024-12-05T22:19:05,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 633 msec 2024-12-05T22:19:05,497 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733437145497, completionTime=-1 2024-12-05T22:19:05,497 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T22:19:05,497 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-05T22:19:05,498 DEBUG [hconnection-0x6411c697-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:19:05,499 INFO [RS-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37194, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:19:05,500 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-05T22:19:05,500 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733437205500 2024-12-05T22:19:05,500 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733437265500 2024-12-05T22:19:05,500 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec 2024-12-05T22:19:05,505 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45153,1733437144639-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:05,505 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45153,1733437144639-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:05,505 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45153,1733437144639-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:05,505 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d029b80fd32e:45153, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:05,505 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:05,505 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-05T22:19:05,505 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T22:19:05,506 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-05T22:19:05,506 DEBUG [master/d029b80fd32e:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-05T22:19:05,507 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T22:19:05,507 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:05,508 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T22:19:05,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:19:05,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741835_1011 (size=358) 2024-12-05T22:19:05,516 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a4eaa6c01fead4448f59b0cb5c46c9c4, NAME => 'hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394 2024-12-05T22:19:05,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:19:05,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741836_1012 (size=42) 2024-12-05T22:19:05,522 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:19:05,522 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing a4eaa6c01fead4448f59b0cb5c46c9c4, disabling compactions & flushes 2024-12-05T22:19:05,522 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:05,522 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:05,522 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. after waiting 0 ms 2024-12-05T22:19:05,523 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:05,523 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:05,523 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for a4eaa6c01fead4448f59b0cb5c46c9c4: 2024-12-05T22:19:05,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T22:19:05,524 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733437145524"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733437145524"}]},"ts":"1733437145524"} 2024-12-05T22:19:05,526 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T22:19:05,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T22:19:05,526 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733437145526"}]},"ts":"1733437145526"} 2024-12-05T22:19:05,527 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-05T22:19:05,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=a4eaa6c01fead4448f59b0cb5c46c9c4, ASSIGN}] 2024-12-05T22:19:05,533 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=a4eaa6c01fead4448f59b0cb5c46c9c4, ASSIGN 2024-12-05T22:19:05,533 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=a4eaa6c01fead4448f59b0cb5c46c9c4, ASSIGN; state=OFFLINE, location=d029b80fd32e,33085,1733437144693; forceNewPlan=false, retain=false 2024-12-05T22:19:05,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:19:05,684 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=a4eaa6c01fead4448f59b0cb5c46c9c4, regionState=OPENING, regionLocation=d029b80fd32e,33085,1733437144693 2024-12-05T22:19:05,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure a4eaa6c01fead4448f59b0cb5c46c9c4, server=d029b80fd32e,33085,1733437144693}] 2024-12-05T22:19:05,838 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d029b80fd32e,33085,1733437144693 2024-12-05T22:19:05,841 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:05,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => a4eaa6c01fead4448f59b0cb5c46c9c4, NAME => 'hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4.', STARTKEY => '', ENDKEY => ''} 2024-12-05T22:19:05,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace a4eaa6c01fead4448f59b0cb5c46c9c4 2024-12-05T22:19:05,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T22:19:05,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for a4eaa6c01fead4448f59b0cb5c46c9c4 2024-12-05T22:19:05,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for a4eaa6c01fead4448f59b0cb5c46c9c4 2024-12-05T22:19:05,843 INFO [StoreOpener-a4eaa6c01fead4448f59b0cb5c46c9c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a4eaa6c01fead4448f59b0cb5c46c9c4 2024-12-05T22:19:05,844 INFO [StoreOpener-a4eaa6c01fead4448f59b0cb5c46c9c4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4eaa6c01fead4448f59b0cb5c46c9c4 columnFamilyName info 2024-12-05T22:19:05,844 DEBUG [StoreOpener-a4eaa6c01fead4448f59b0cb5c46c9c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T22:19:05,844 INFO [StoreOpener-a4eaa6c01fead4448f59b0cb5c46c9c4-1 {}] regionserver.HStore(327): Store=a4eaa6c01fead4448f59b0cb5c46c9c4/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T22:19:05,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/namespace/a4eaa6c01fead4448f59b0cb5c46c9c4 2024-12-05T22:19:05,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/namespace/a4eaa6c01fead4448f59b0cb5c46c9c4 2024-12-05T22:19:05,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for a4eaa6c01fead4448f59b0cb5c46c9c4 2024-12-05T22:19:05,848 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/namespace/a4eaa6c01fead4448f59b0cb5c46c9c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T22:19:05,849 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened a4eaa6c01fead4448f59b0cb5c46c9c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=693123, jitterRate=-0.11864903569221497}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T22:19:05,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for a4eaa6c01fead4448f59b0cb5c46c9c4: 2024-12-05T22:19:05,850 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4., pid=6, masterSystemTime=1733437145838 2024-12-05T22:19:05,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:05,851 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:05,852 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=a4eaa6c01fead4448f59b0cb5c46c9c4, regionState=OPEN, openSeqNum=2, regionLocation=d029b80fd32e,33085,1733437144693 2024-12-05T22:19:05,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T22:19:05,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure a4eaa6c01fead4448f59b0cb5c46c9c4, server=d029b80fd32e,33085,1733437144693 in 167 msec 2024-12-05T22:19:05,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T22:19:05,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=a4eaa6c01fead4448f59b0cb5c46c9c4, ASSIGN in 323 msec 2024-12-05T22:19:05,857 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T22:19:05,857 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733437145857"}]},"ts":"1733437145857"} 2024-12-05T22:19:05,858 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-05T22:19:05,861 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T22:19:05,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 356 msec 2024-12-05T22:19:05,907 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-05T22:19:05,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:19:05,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:05,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:05,912 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-05T22:19:05,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:19:05,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 9 msec 2024-12-05T22:19:05,924 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-05T22:19:05,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T22:19:05,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 8 msec 2024-12-05T22:19:05,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-05T22:19:05,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-05T22:19:05,946 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.223sec 2024-12-05T22:19:05,946 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T22:19:05,946 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T22:19:05,946 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T22:19:05,946 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T22:19:05,946 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T22:19:05,946 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45153,1733437144639-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T22:19:05,946 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45153,1733437144639-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T22:19:05,948 DEBUG [master/d029b80fd32e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-05T22:19:05,948 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T22:19:05,948 INFO [master/d029b80fd32e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d029b80fd32e,45153,1733437144639-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T22:19:06,018 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x581fea79 to 127.0.0.1:53795 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e98152b 2024-12-05T22:19:06,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14374f38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T22:19:06,024 DEBUG [hconnection-0x79f4cbbd-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T22:19:06,026 INFO [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37202, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T22:19:06,027 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=d029b80fd32e,45153,1733437144639 2024-12-05T22:19:06,027 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T22:19:06,029 INFO [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false 2024-12-05T22:19:06,029 INFO [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-05T22:19:06,031 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/test.com,8080,1, archiveDir=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/oldWALs, maxLogs=32 2024-12-05T22:19:06,032 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733437146032 2024-12-05T22:19:06,036 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/test.com,8080,1/test.com%2C8080%2C1.1733437146032 2024-12-05T22:19:06,036 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36453:36453),(127.0.0.1/127.0.0.1:45941:45941)] 2024-12-05T22:19:06,036 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733437146036 2024-12-05T22:19:06,042 INFO [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/test.com,8080,1/test.com%2C8080%2C1.1733437146032 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/test.com,8080,1/test.com%2C8080%2C1.1733437146036 2024-12-05T22:19:06,043 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45941:45941),(127.0.0.1/127.0.0.1:36453:36453)] 2024-12-05T22:19:06,043 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/test.com,8080,1/test.com%2C8080%2C1.1733437146032 is not closed yet, will try archiving it next time 2024-12-05T22:19:06,043 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/test.com,8080,1 2024-12-05T22:19:06,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741838_1014 (size=93) 2024-12-05T22:19:06,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741837_1013 (size=93) 2024-12-05T22:19:06,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741837_1013 (size=93) 2024-12-05T22:19:06,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741838_1014 (size=93) 2024-12-05T22:19:06,046 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/test.com,8080,1/test.com%2C8080%2C1.1733437146032 to hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/oldWALs/test.com%2C8080%2C1.1733437146032 2024-12-05T22:19:06,049 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/oldWALs 2024-12-05T22:19:06,049 INFO [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733437146036) 2024-12-05T22:19:06,050 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-05T22:19:06,050 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x581fea79 to 127.0.0.1:53795 2024-12-05T22:19:06,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:06,050 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T22:19:06,050 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=928136160, stopped=false 2024-12-05T22:19:06,050 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=d029b80fd32e,45153,1733437144639 2024-12-05T22:19:06,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:19:06,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T22:19:06,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:06,052 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-05T22:19:06,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:06,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:06,052 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:19:06,052 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T22:19:06,052 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,33085,1733437144693' ***** 2024-12-05T22:19:06,052 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-05T22:19:06,053 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T22:19:06,053 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-05T22:19:06,053 INFO [RS:0;d029b80fd32e:33085 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T22:19:06,053 INFO [RS:0;d029b80fd32e:33085 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T22:19:06,053 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(3579): Received CLOSE for a4eaa6c01fead4448f59b0cb5c46c9c4 2024-12-05T22:19:06,053 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,33085,1733437144693 2024-12-05T22:19:06,053 DEBUG [RS:0;d029b80fd32e:33085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:06,054 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T22:19:06,054 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T22:19:06,054 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T22:19:06,054 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-05T22:19:06,054 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a4eaa6c01fead4448f59b0cb5c46c9c4, disabling compactions & flushes 2024-12-05T22:19:06,054 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:06,054 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:06,054 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. after waiting 0 ms 2024-12-05T22:19:06,054 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:06,054 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-12-05T22:19:06,054 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a4eaa6c01fead4448f59b0cb5c46c9c4 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-05T22:19:06,054 DEBUG [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1603): Online Regions={a4eaa6c01fead4448f59b0cb5c46c9c4=hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4., 1588230740=hbase:meta,,1.1588230740} 2024-12-05T22:19:06,054 DEBUG [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, a4eaa6c01fead4448f59b0cb5c46c9c4 2024-12-05T22:19:06,054 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T22:19:06,054 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T22:19:06,054 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T22:19:06,054 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T22:19:06,054 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T22:19:06,054 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB 2024-12-05T22:19:06,076 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/namespace/a4eaa6c01fead4448f59b0cb5c46c9c4/.tmp/info/adfd58eb6d124884ba9f26bb2a8d2c3c is 45, key is default/info:d/1733437145916/Put/seqid=0 2024-12-05T22:19:06,076 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/.tmp/info/fb0c91f0ee97433587700f6c941230f7 is 143, key is hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4./info:regioninfo/1733437145852/Put/seqid=0 2024-12-05T22:19:06,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741839_1015 (size=5037) 2024-12-05T22:19:06,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741840_1016 (size=6595) 2024-12-05T22:19:06,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741839_1015 (size=5037) 2024-12-05T22:19:06,085 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/namespace/a4eaa6c01fead4448f59b0cb5c46c9c4/.tmp/info/adfd58eb6d124884ba9f26bb2a8d2c3c 2024-12-05T22:19:06,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741840_1016 (size=6595) 2024-12-05T22:19:06,088 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/.tmp/info/fb0c91f0ee97433587700f6c941230f7 2024-12-05T22:19:06,092 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/namespace/a4eaa6c01fead4448f59b0cb5c46c9c4/.tmp/info/adfd58eb6d124884ba9f26bb2a8d2c3c as hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/namespace/a4eaa6c01fead4448f59b0cb5c46c9c4/info/adfd58eb6d124884ba9f26bb2a8d2c3c 2024-12-05T22:19:06,097 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/namespace/a4eaa6c01fead4448f59b0cb5c46c9c4/info/adfd58eb6d124884ba9f26bb2a8d2c3c, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T22:19:06,098 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for a4eaa6c01fead4448f59b0cb5c46c9c4 in 44ms, sequenceid=6, compaction requested=false 2024-12-05T22:19:06,102 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/namespace/a4eaa6c01fead4448f59b0cb5c46c9c4/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T22:19:06,102 INFO [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:06,102 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a4eaa6c01fead4448f59b0cb5c46c9c4: 2024-12-05T22:19:06,102 DEBUG [RS_CLOSE_REGION-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733437145505.a4eaa6c01fead4448f59b0cb5c46c9c4. 2024-12-05T22:19:06,115 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/.tmp/table/a50842ad5f384067a933b22a62eb2256 is 51, key is hbase:namespace/table:state/1733437145857/Put/seqid=0 2024-12-05T22:19:06,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741841_1017 (size=5242) 2024-12-05T22:19:06,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741841_1017 (size=5242) 2024-12-05T22:19:06,120 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/.tmp/table/a50842ad5f384067a933b22a62eb2256 2024-12-05T22:19:06,126 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/.tmp/info/fb0c91f0ee97433587700f6c941230f7 as hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/info/fb0c91f0ee97433587700f6c941230f7 2024-12-05T22:19:06,130 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/info/fb0c91f0ee97433587700f6c941230f7, entries=10, sequenceid=9, filesize=6.4 K 2024-12-05T22:19:06,130 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/.tmp/table/a50842ad5f384067a933b22a62eb2256 as hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/table/a50842ad5f384067a933b22a62eb2256 2024-12-05T22:19:06,135 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/table/a50842ad5f384067a933b22a62eb2256, entries=2, sequenceid=9, filesize=5.1 K 2024-12-05T22:19:06,135 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=9, compaction requested=false 2024-12-05T22:19:06,139 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-05T22:19:06,140 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T22:19:06,140 INFO [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T22:19:06,140 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T22:19:06,140 DEBUG [RS_CLOSE_META-regionserver/d029b80fd32e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T22:19:06,254 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,33085,1733437144693; all regions closed. 2024-12-05T22:19:06,254 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/d029b80fd32e,33085,1733437144693 2024-12-05T22:19:06,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741834_1010 (size=2484) 2024-12-05T22:19:06,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741834_1010 (size=2484) 2024-12-05T22:19:06,258 DEBUG [RS:0;d029b80fd32e:33085 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/oldWALs 2024-12-05T22:19:06,258 INFO [RS:0;d029b80fd32e:33085 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog d029b80fd32e%2C33085%2C1733437144693.meta:.meta(num 1733437145463) 2024-12-05T22:19:06,259 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/WALs/d029b80fd32e,33085,1733437144693 2024-12-05T22:19:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741833_1009 (size=1414) 2024-12-05T22:19:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741833_1009 (size=1414) 2024-12-05T22:19:06,262 DEBUG [RS:0;d029b80fd32e:33085 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/oldWALs 2024-12-05T22:19:06,262 INFO [RS:0;d029b80fd32e:33085 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog d029b80fd32e%2C33085%2C1733437144693:(num 1733437145118) 2024-12-05T22:19:06,262 DEBUG [RS:0;d029b80fd32e:33085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:06,262 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T22:19:06,263 INFO [RS:0;d029b80fd32e:33085 {}] hbase.ChoreService(370): Chore service for: regionserver/d029b80fd32e:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T22:19:06,263 INFO [regionserver/d029b80fd32e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:19:06,263 INFO [RS:0;d029b80fd32e:33085 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:33085 2024-12-05T22:19:06,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d029b80fd32e,33085,1733437144693 2024-12-05T22:19:06,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T22:19:06,267 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d029b80fd32e,33085,1733437144693] 2024-12-05T22:19:06,267 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing d029b80fd32e,33085,1733437144693; numProcessing=1 2024-12-05T22:19:06,269 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/d029b80fd32e,33085,1733437144693 already deleted, retry=false 2024-12-05T22:19:06,269 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; d029b80fd32e,33085,1733437144693 expired; onlineServers=0 2024-12-05T22:19:06,269 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd029b80fd32e,45153,1733437144639' ***** 2024-12-05T22:19:06,269 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T22:19:06,269 DEBUG [M:0;d029b80fd32e:45153 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efa5e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d029b80fd32e/172.17.0.2:0 2024-12-05T22:19:06,269 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HRegionServer(1224): stopping server d029b80fd32e,45153,1733437144639 2024-12-05T22:19:06,269 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HRegionServer(1250): stopping server d029b80fd32e,45153,1733437144639; all regions closed. 2024-12-05T22:19:06,269 DEBUG [M:0;d029b80fd32e:45153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T22:19:06,269 DEBUG [M:0;d029b80fd32e:45153 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T22:19:06,269 DEBUG [M:0;d029b80fd32e:45153 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T22:19:06,269 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T22:19:06,269 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733437144868 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.small.0-1733437144868,5,FailOnTimeoutGroup] 2024-12-05T22:19:06,269 DEBUG [master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733437144868 {}] cleaner.HFileCleaner(306): Exit Thread[master/d029b80fd32e:0:becomeActiveMaster-HFileCleaner.large.0-1733437144868,5,FailOnTimeoutGroup] 2024-12-05T22:19:06,269 INFO [M:0;d029b80fd32e:45153 {}] hbase.ChoreService(370): Chore service for: master/d029b80fd32e:0 had [] on shutdown 2024-12-05T22:19:06,270 DEBUG [M:0;d029b80fd32e:45153 {}] master.HMaster(1733): Stopping service threads 2024-12-05T22:19:06,270 INFO [M:0;d029b80fd32e:45153 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T22:19:06,270 INFO [M:0;d029b80fd32e:45153 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T22:19:06,270 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T22:19:06,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T22:19:06,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T22:19:06,271 DEBUG [M:0;d029b80fd32e:45153 {}] zookeeper.ZKUtil(347): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T22:19:06,271 WARN [M:0;d029b80fd32e:45153 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T22:19:06,271 INFO [M:0;d029b80fd32e:45153 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-05T22:19:06,271 INFO [M:0;d029b80fd32e:45153 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T22:19:06,271 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T22:19:06,271 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:06,271 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:06,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T22:19:06,271 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T22:19:06,271 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:06,271 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.32 KB heapSize=32.31 KB 2024-12-05T22:19:06,287 DEBUG [M:0;d029b80fd32e:45153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/433ddfb0f1124f449c8eef451f2dde0b is 82, key is hbase:meta,,1/info:regioninfo/1733437145484/Put/seqid=0 2024-12-05T22:19:06,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741842_1018 (size=5672) 2024-12-05T22:19:06,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741842_1018 (size=5672) 2024-12-05T22:19:06,292 INFO [M:0;d029b80fd32e:45153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/433ddfb0f1124f449c8eef451f2dde0b 2024-12-05T22:19:06,309 DEBUG [M:0;d029b80fd32e:45153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/62552f6637e34ec4902e0da33e14fce6 is 696, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733437145862/Put/seqid=0 2024-12-05T22:19:06,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741843_1019 (size=6626) 2024-12-05T22:19:06,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741843_1019 (size=6626) 2024-12-05T22:19:06,314 INFO [M:0;d029b80fd32e:45153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.72 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/62552f6637e34ec4902e0da33e14fce6 2024-12-05T22:19:06,334 DEBUG [M:0;d029b80fd32e:45153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/622e5299f5ff436da8f3085736ace1c6 is 69, key is d029b80fd32e,33085,1733437144693/rs:state/1733437144958/Put/seqid=0 2024-12-05T22:19:06,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741844_1020 (size=5156) 2024-12-05T22:19:06,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741844_1020 (size=5156) 2024-12-05T22:19:06,346 INFO [M:0;d029b80fd32e:45153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/622e5299f5ff436da8f3085736ace1c6 2024-12-05T22:19:06,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:19:06,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33085-0x10184b5a8be0001, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:19:06,368 INFO [RS:0;d029b80fd32e:33085 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,33085,1733437144693; zookeeper connection closed. 2024-12-05T22:19:06,368 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@697d847c {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@697d847c 2024-12-05T22:19:06,368 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T22:19:06,375 DEBUG [M:0;d029b80fd32e:45153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/999bc22c0d5949e2943d6aa6d6f6ecfa is 52, key is load_balancer_on/state:d/1733437146029/Put/seqid=0 2024-12-05T22:19:06,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741845_1021 (size=5056) 2024-12-05T22:19:06,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741845_1021 (size=5056) 2024-12-05T22:19:06,383 INFO [M:0;d029b80fd32e:45153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/999bc22c0d5949e2943d6aa6d6f6ecfa 2024-12-05T22:19:06,390 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/433ddfb0f1124f449c8eef451f2dde0b as hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/433ddfb0f1124f449c8eef451f2dde0b 2024-12-05T22:19:06,396 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/433ddfb0f1124f449c8eef451f2dde0b, entries=8, sequenceid=70, filesize=5.5 K 2024-12-05T22:19:06,397 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/62552f6637e34ec4902e0da33e14fce6 as hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/62552f6637e34ec4902e0da33e14fce6 2024-12-05T22:19:06,402 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/62552f6637e34ec4902e0da33e14fce6, entries=8, sequenceid=70, filesize=6.5 K 2024-12-05T22:19:06,403 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/622e5299f5ff436da8f3085736ace1c6 as hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/622e5299f5ff436da8f3085736ace1c6 2024-12-05T22:19:06,407 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/622e5299f5ff436da8f3085736ace1c6, entries=1, sequenceid=70, filesize=5.0 K 2024-12-05T22:19:06,408 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/999bc22c0d5949e2943d6aa6d6f6ecfa as hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/999bc22c0d5949e2943d6aa6d6f6ecfa 2024-12-05T22:19:06,413 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40725/user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/999bc22c0d5949e2943d6aa6d6f6ecfa, entries=1, sequenceid=70, filesize=4.9 K 2024-12-05T22:19:06,415 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HRegion(3040): Finished flush of dataSize ~25.32 KB/25929, heapSize ~32.25 KB/33024, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 143ms, sequenceid=70, compaction requested=false 2024-12-05T22:19:06,417 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T22:19:06,417 DEBUG [M:0;d029b80fd32e:45153 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T22:19:06,420 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5a6ddac8-f7c6-7f4a-f36e-e53c09b2f394/MasterData/WALs/d029b80fd32e,45153,1733437144639 2024-12-05T22:19:06,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41825 is added to blk_1073741830_1006 (size=31030) 2024-12-05T22:19:06,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44207 is added to blk_1073741830_1006 (size=31030) 2024-12-05T22:19:06,423 INFO [M:0;d029b80fd32e:45153 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-05T22:19:06,423 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T22:19:06,423 INFO [M:0;d029b80fd32e:45153 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45153 2024-12-05T22:19:06,425 DEBUG [M:0;d029b80fd32e:45153 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/d029b80fd32e,45153,1733437144639 already deleted, retry=false 2024-12-05T22:19:06,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:19:06,526 INFO [M:0;d029b80fd32e:45153 {}] regionserver.HRegionServer(1307): Exiting; stopping=d029b80fd32e,45153,1733437144639; zookeeper connection closed. 2024-12-05T22:19:06,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45153-0x10184b5a8be0000, quorum=127.0.0.1:53795, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T22:19:06,529 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420757c0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:19:06,530 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7ef49dde{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:19:06,530 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:19:06,530 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33d3a777{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:19:06,530 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69321fa9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/hadoop.log.dir/,STOPPED} 2024-12-05T22:19:06,533 WARN [BP-1006552126-172.17.0.2-1733437143795 heartbeating to localhost/127.0.0.1:40725 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:19:06,533 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:19:06,533 WARN [BP-1006552126-172.17.0.2-1733437143795 heartbeating to localhost/127.0.0.1:40725 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1006552126-172.17.0.2-1733437143795 (Datanode Uuid e43499f8-3343-46ad-949c-181fe36cc1a2) service to localhost/127.0.0.1:40725 2024-12-05T22:19:06,533 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:19:06,534 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/dfs/data/data4/current/BP-1006552126-172.17.0.2-1733437143795 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:19:06,535 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:19:06,536 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/dfs/data/data3/current/BP-1006552126-172.17.0.2-1733437143795 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:19:06,538 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@37530d4a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T22:19:06,538 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5daa6290{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:19:06,538 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:19:06,538 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2990422d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:19:06,538 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d6c116b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/hadoop.log.dir/,STOPPED} 2024-12-05T22:19:06,544 WARN [BP-1006552126-172.17.0.2-1733437143795 heartbeating to localhost/127.0.0.1:40725 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T22:19:06,545 WARN [BP-1006552126-172.17.0.2-1733437143795 heartbeating to localhost/127.0.0.1:40725 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1006552126-172.17.0.2-1733437143795 (Datanode Uuid 6c92c92f-7698-4d2e-b6c2-af6797079ce7) service to localhost/127.0.0.1:40725 2024-12-05T22:19:06,545 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T22:19:06,545 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/dfs/data/data1/current/BP-1006552126-172.17.0.2-1733437143795 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:19:06,545 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T22:19:06,545 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/cluster_a0c973ad-0d3d-4d36-fce9-858848b65891/dfs/data/data2/current/BP-1006552126-172.17.0.2-1733437143795 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T22:19:06,546 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T22:19:06,565 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a3e1974{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T22:19:06,566 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@63867263{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T22:19:06,566 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T22:19:06,566 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@693f2ee7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T22:19:06,567 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6db6e54e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c74698b7-67d4-9f26-527e-6f4cac950cf7/hadoop.log.dir/,STOPPED} 2024-12-05T22:19:06,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36083/user/jenkins/test-data/0e9e3429-fbcd-45d4-f939-69e757b7c0be/WALs/d029b80fd32e,45395,1733436864508/d029b80fd32e%2C45395%2C1733436864508.meta.1733436865292.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-05T22:19:06,585 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-05T22:19:06,605 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2024-12-05T22:19:06,617 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=148 (was 126) - Thread LEAK? -, OpenFileDescriptor=518 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=238 (was 238), ProcessCount=11 (was 11), AvailableMemoryMB=3485 (was 3586)